Commit e6cfda83 authored by Jonathan Mace's avatar Jonathan Mace
Browse files

Uncommitted experiment changes

parent f774b723
......@@ -85,6 +85,11 @@ void printUsage() {
<< "\t\t Workload parameters:" << std::endl
<< "\t\t\t num_models: (int, default 15) the number of models you're using" << std::endl
<< "\t\t\t total_requests: (int, default 1000) the total requests across all models, per second" << std::endl
<< "\t comparison_experiment2" << std::endl
<< "\t\t Description: closed-loop version of comparison experiment" << std::endl
<< "\t\t Workload parameters:" << std::endl
<< "\t\t\t num_models: (int, default 15) the number of models you're using" << std::endl
<< "\t\t\t concurrency: (int, default 16) closed loop workload concurrency" << std::endl
<< "\t azure" << std::endl
<< "\t\t Description: replay an azure workload trace. Can be run with no arguments, in which case default values are used. The defaults will load 3100 models and replay a trace that will give approximately the total load the system can handle." << std::endl
<< "\t\t Workload parameters:" << std::endl
......@@ -200,6 +205,11 @@ int main(int argc, char *argv[])
unsigned num_models = argc > ++i ? atoi(argv[i]) : 15;
unsigned total_requests = argc > ++i ? atoi(argv[i]) : 1000;
engine = workload::comparison_experiment(client, num_models, total_requests);
} else if (workload == "comparison_experiment2") {
int i = 2;
unsigned num_models = argc > ++i ? atoi(argv[i]) : 15;
unsigned concurrency = argc > ++i ? atoi(argv[i]) : 16;
engine = workload::comparison_experiment_closed(client, num_models, concurrency);
} else if (workload == "azure_half")
engine = workload::azure_parameterized(client);
else if (workload == "azure_small")
......@@ -209,7 +219,7 @@ int main(int argc, char *argv[])
else if (workload == "azure_fast")
engine = workload::azure_fast(client);
else if (workload == "bursty_experiment")
engine = workload::bursty_experiment(client);
engine = workload::bursty_experiment2(client);
else {
std::cout << "Unknown workload " << workload << std::endl << std::endl;
printUsage();
......
......@@ -35,6 +35,26 @@ Engine* comparison_experiment(clockwork::Client* client,
return engine;
}
Engine* comparison_experiment_closed(clockwork::Client* client,
int num_models=15,
int concurrency=16) {
Engine* engine = new Engine();
std::string modelpath = util::get_clockwork_modelzoo()["resnet50_v2"];
auto models = client->load_remote_models(modelpath, num_models);
for (int i = 0; i < num_models; i++) {
models[i]->disable_inputs();
engine->AddWorkload(new ClosedLoop(
i, // client id, just use the same for this
models[i], // model
concurrency // concurrency
));
}
return engine;
}
Engine* fill_memory(clockwork::Client* client) {
Engine* engine = new Engine();
......@@ -451,56 +471,51 @@ Engine* azure_fast(clockwork::Client* client, unsigned trace_id = 1) {
}
Engine* bursty_experiment2(
clockwork::Client* client,
unsigned trace_id = 1,
bool mul = false,
unsigned num_models = 3600,
unsigned intervals = 30,
unsigned interval_duration_seconds = 20,
unsigned num_gpus = 8,
unsigned work_per_gpu = 1000
clockwork::Client* client,
unsigned start_at = 1,
unsigned increment = 1,
unsigned stop_at = 3600,
unsigned interval_duration_seconds = 1, // seconds
unsigned lead_in = 300, // lead-in intervals
unsigned total_request_rate = 1000
) {
Engine* engine = new Engine();
std::string modelpath = util::get_clockwork_modelzoo()["resnet50_v2"];
unsigned total_request_rate = work_per_gpu * num_gpus;
std::vector<std::vector<unsigned>> interval_rates(stop_at);
unsigned num_models = stop_at;
std::vector<std::vector<unsigned>> interval_rates;
for (unsigned i = 0; i < num_models; i++) {
interval_rates.push_back(std::vector<unsigned>(intervals, 0));
interval_rates[i].resize(lead_in + (stop_at - start_at) / increment, 0);
}
for (unsigned i = 0; i < intervals; i++) {
unsigned models_this_interval;
if (mul) {
models_this_interval = std::min((unsigned) round(std::pow(2, i/2.0)), num_models);
} else {
models_this_interval = (i + 1) * num_models / intervals;
}
std::cout << "Interval " << i << " " << models_this_interval << " models" << std::endl;
unsigned interval = lead_in;
while (start_at <= stop_at) {
for (int j = 0; j < 60 * total_request_rate; j++) {
unsigned model = j % models_this_interval;
interval_rates[model][i]++;
unsigned model = j % start_at;
interval_rates[model][interval]++;
}
start_at += increment;
interval++;
}
for (unsigned i = 0; i < 5; i++) {
std::cout << "Model " << i;
for (unsigned j = 0; j < intervals; j++) {
std::cout << " " << interval_rates[i][j];
for (auto rate : interval_rates[i]) {
std::cout << " " << rate;
}
std::cout << std::endl;
}
std::cout << "..." << std::endl;
for (unsigned i = num_models - 6; i < num_models; i++) {
std::cout << "Model " << i;
for (unsigned j = 0; j < intervals; j++) {
std::cout << " " << interval_rates[i][j];
for (auto rate : interval_rates[i]) {
std::cout << " " << rate;
}
std::cout << std::endl;
}
std::string modelpath = util::get_clockwork_modelzoo()["resnet50_v2"];
std::vector<Model*> models;
while (models.size() < num_models) {
unsigned to_load = std::min((int) (num_models - models.size()), 100);
......@@ -510,6 +525,8 @@ Engine* bursty_experiment2(
}
}
Model* hipri = client->load_remote_model(modelpath);
for (unsigned i = 0; i < models.size(); i++) {
auto model = models[i];
model->disable_inputs();
......@@ -528,6 +545,13 @@ Engine* bursty_experiment2(
engine->AddWorkload(replay);
}
engine->AddWorkload(new PoissonOpenLoop(
models.size(), // client id
hipri, // model
models.size(), // rng seed
200 // requests/second
));
return engine;
}
......@@ -792,7 +816,7 @@ Engine* azure2(clockwork::Client* client,
copies_per_model = num_workers * 402;
} else if (memory_load_factor == 4) {
if (use_all_models)
copies_per_model = 50;
copies_per_model = 66;
else
copies_per_model = 3600;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment