Skip to content

Commit d6f3f18

Browse files
PaliCPaliC
committed
dynamo benchmarking
ghstack-source-id: 902145d48118591188d0f0b72e1043152e8a6d2f Pull Request resolved: #265
1 parent 22ff1ca commit d6f3f18

File tree

5 files changed

+161
-24
lines changed

5 files changed

+161
-24
lines changed

‎multipy/runtime/CMakeLists.txt

+12-3
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,9 @@ set(INTERPRETER_TEST_SOURCES
8787
set(INTERPRETER_TEST_SOURCES_GPU
8888
${DEPLOY_DIR}/test_deploy_gpu.cpp
8989
)
90-
91-
# TODO: Currently tests can only be done when ABI=1 as the testing infrustructure
92-
# used by ASSERT_TRUE requires ABI=1 in Github actions, we should fix this!
90+
set(INTERPRETER_TEST_SOURCES_COMPAT
91+
${DEPLOY_DIR}/test_deploy_compat.cpp
92+
)
9393

9494
add_executable(test_deploy ${INTERPRETER_TEST_SOURCES})
9595
# target_compile_definitions(test_deploy PUBLIC TEST_CUSTOM_LIBRARY)
@@ -99,6 +99,15 @@ target_link_libraries(test_deploy
9999
)
100100
target_include_directories(test_deploy PRIVATE ${CMAKE_SOURCE_DIR}/../..)
101101

102+
add_executable(test_deploy_compat ${INTERPRETER_TEST_SOURCES_COMPAT})
103+
# target_compile_definitions(test_deploy_compat PUBLIC TEST_CUSTOM_LIBRARY)
104+
target_include_directories(test_deploy_compat PRIVATE ${PYTORCH_ROOT}/torch)
105+
target_link_libraries(test_deploy_compat
106+
PUBLIC "-Wl,--no-as-needed -rdynamic" gtest dl torch_deploy_interface c10 torch_cpu
107+
)
108+
target_include_directories(test_deploy_compat PRIVATE ${CMAKE_SOURCE_DIR}/../..)
109+
110+
102111
if(BUILD_CUDA_TESTS)
103112
LINK_DIRECTORIES("${PYTORCH_ROOT}/torch/lib")
104113
add_executable(test_deploy_gpu ${INTERPRETER_TEST_SOURCES_GPU})

‎multipy/runtime/example/benchmark.cpp

+12-3
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ struct Benchmark {
178178
manager.debugLimitInterpreters(1);
179179
} else if (strategy == "multi_python") {
180180
manager.debugLimitInterpreters(n_threads_);
181+
} else if (strategy == "dynamo+deploy"){
182+
manager.debugLimitInterpreters(n_threads_);
181183
}
182184
}
183185

@@ -295,6 +297,7 @@ int main(int argc, char* argv[]) {
295297
cuda = std::string(argv[2]) == "cuda";
296298
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
297299
bool jit_enable = std::string(argv[3]) == "jit";
300+
bool inductor_enable = std::string(argv[3]) == "inductor";
298301
Report::report_header(std::cout);
299302
torch::deploy::InterpreterManager manager(max_thread);
300303

@@ -311,19 +314,25 @@ int main(int argc, char* argv[]) {
311314
if (n_thread > max_thread) {
312315
continue;
313316
}
314-
for (std::string strategy : {"one_python", "multi_python", "jit"}) {
317+
for (std::string strategy : {"one_python", "multi_python", "jit", "dynamo+deploy"}) {
315318
if (strategy == "jit") {
316319
if (!jit_enable) {
317320
continue;
318321
}
319322
if (!exists(model_file + "_jit")) {
320323
continue;
321324
}
322-
}
323-
if (strategy == "one_python") {
325+
} else if (strategy == "one_python") {
324326
Benchmark b(manager, 1, strategy, model_file);
325327
Report r = b.run();
326328
r.report(std::cout);
329+
} else if (strategy == "dynamo+deploy") {
330+
if (!exists(model_file + "_dynamo")) {
331+
continue;
332+
}
333+
Benchmark b(manager, n_thread, strategy, model_file + "_dynamo");
334+
Report r = b.run();
335+
r.report(std::cout);
327336
} else {
328337
Benchmark b(manager, n_thread, strategy, model_file);
329338
Report r = b.run();

‎multipy/runtime/example/examples.py

+5
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import torch
1010
import torch.nn as nn
1111
from torch import Tensor
12+
import torch._dynamo as torchdynamo
1213

1314

1415
class Simple(torch.nn.Module):
@@ -128,6 +129,10 @@ def forward(self, x):
128129
def resnet18():
129130
return ResNet(BasicBlock, [2, 2, 2, 2])
130131

132+
@torchdynamo.optimize("ts")
133+
def resnet18_dynamo():
134+
return resnet18();
135+
131136

132137
class BatchedModel(nn.Module):
133138
def forward(self, input1: Tensor, input2: Tensor) -> Tuple[Tensor, Tensor]:

‎multipy/runtime/example/generate_examples.py

+27-18
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
multi_return_metadata,
2222
MultiReturn,
2323
resnet18,
24+
resnet18_dynamo,
2425
Simple,
2526
)
2627
except ImportError:
@@ -30,6 +31,7 @@
3031
multi_return_metadata,
3132
MultiReturn,
3233
resnet18,
34+
resnet18_dynamo,
3335
Simple,
3436
)
3537

@@ -60,34 +62,39 @@ def save(
6062
name,
6163
model,
6264
model_jit=None,
65+
model_dynamo=None,
6366
eg=None,
6467
featurestore_meta=None,
6568
text_in_extra_file=None,
6669
binary_in_extra_file=None,
6770
):
68-
with PackageExporter(str(p / name)) as e:
69-
e.mock("iopath.**")
70-
e.intern("**")
71-
e.save_pickle("model", "model.pkl", model)
72-
if eg:
73-
e.save_pickle("model", "example.pkl", eg)
74-
if featurestore_meta:
75-
# TODO(whc) can this name come from buck somehow,
76-
# so it's consistent with predictor_config_constants::METADATA_FILE_NAME()?
77-
e.save_text("extra_files", "metadata.json", featurestore_meta)
78-
if text_in_extra_file:
79-
e.save_text("extra_files", "text", text_in_extra_file)
80-
if binary_in_extra_file:
81-
e.save_binary("extra_files", "binary", binary_in_extra_file)
82-
71+
def package_model(name, model):
72+
with PackageExporter(str(p / name)) as e:
73+
e.mock("iopath.**")
74+
e.intern("**")
75+
e.save_pickle("model", "model.pkl", model)
76+
if eg:
77+
e.save_pickle("model", "example.pkl", eg)
78+
if featurestore_meta:
79+
# TODO(whc) can this name come from buck somehow,
80+
# so it's consistent with predictor_config_constants::METADATA_FILE_NAME()?
81+
e.save_text("extra_files", "metadata.json", featurestore_meta)
82+
if text_in_extra_file:
83+
e.save_text("extra_files", "text", text_in_extra_file)
84+
if binary_in_extra_file:
85+
e.save_binary("extra_files", "binary", binary_in_extra_file)
86+
87+
package_model(name, model)
88+
if model_dynamo:
89+
package_model(name + "_dynamo", model_dynamo)
8390
if model_jit:
8491
model_jit.save(str(p / (name + "_jit")))
92+
8593

8694

8795
parser = argparse.ArgumentParser(description="Generate Examples")
8896
parser.add_argument("--install_dir", help="Root directory for all output files")
8997

90-
9198
if __name__ == "__main__":
9299
args = parser.parse_args()
93100
if args.install_dir is None:
@@ -98,9 +105,10 @@ def save(
98105

99106
resnet = resnet18()
100107
resnet.eval()
108+
resnet_dynamo = resnet18_dynamo()
101109
resnet_eg = torch.rand(1, 3, 224, 224)
102110
resnet_traced = torch.jit.trace(resnet, resnet_eg)
103-
save("resnet", resnet, resnet_traced, (resnet_eg,))
111+
save("resnet", resnet, resnet_traced, resnet_dynamo, (resnet_eg,))
104112

105113
simple = Simple(10, 20)
106114
save(
@@ -117,6 +125,7 @@ def save(
117125
"multi_return",
118126
multi_return,
119127
torch.jit.script(multi_return),
128+
None,
120129
(torch.rand(10, 20),),
121130
multi_return_metadata,
122131
)
@@ -149,4 +158,4 @@ def save(
149158
e.add_dependency("tensorrt")
150159
e.mock("iopath.**")
151160
e.intern("**")
152-
e.save_pickle("make_trt_module", "model.pkl", make_trt_module)
161+
e.save_pickle("make_trt_module", "model.pkl", make_trt_module)

‎multipy/runtime/test_deploy_compat.cpp

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <ATen/Parallel.h>
8+
#include <gtest/gtest.h>
9+
#include <libgen.h>
10+
#include <cstring>
11+
12+
#include <c10/util/irange.h>
13+
#include <libgen.h>
14+
#include <multipy/runtime/deploy.h>
15+
#include <torch/script.h>
16+
#include <torch/torch.h>
17+
18+
#include <future>
19+
#include <iostream>
20+
#include <string>
21+
22+
void compare_torchpy_jit(const char* model_filename, const char* jit_filename) {
23+
// Test
24+
25+
torch::deploy::InterpreterManager m(2);
26+
torch::deploy::Package p = m.loadPackage(model_filename);
27+
auto model = p.loadPickle("model", "model.pkl");
28+
at::IValue eg;
29+
{
30+
auto I = p.acquireSession();
31+
eg = I.self.attr("load_pickle")({"model", "example.pkl"}).toIValue();
32+
}
33+
34+
at::Tensor output = model(eg.toTupleRef().elements()).toTensor();
35+
36+
// Reference
37+
auto ref_model = torch::jit::load(jit_filename);
38+
at::Tensor ref_output =
39+
ref_model.forward(eg.toTupleRef().elements()).toTensor();
40+
ASSERT_TRUE(ref_output.allclose(output, 1e-03, 1e-05));
41+
}
42+
43+
const char* resnet_path = "multipy/runtime/example/generated/resnet_dynamo";
44+
const char* resnet_jit_path = "multipy/runtime/example/generated/resnet_jit";
45+
46+
const char* path(const char* envname, const char* path) {
47+
const char* e = getenv(envname);
48+
return e ? e : path;
49+
}
50+
51+
TEST(TorchpyTest, ResNetWithDynamo) {
52+
compare_torchpy_jit(
53+
path("RESNET", resnet_path),
54+
path("RESNET_JIT", resnet_jit_path));
55+
}
56+
57+
TEST(TorchpyTest, ThreadedResnetModelWithDynamo) {
58+
size_t nthreads = 3;
59+
torch::deploy::InterpreterManager manager(nthreads);
60+
61+
torch::deploy::Package p = manager.loadPackage(path("RESNET", resnet_path));
62+
auto model = p.loadPickle("model", "model.pkl");
63+
auto ref_model = torch::jit::load(path("RESNET_JIT", resnet_jit_path));
64+
65+
auto input = torch::ones({10, 20});
66+
67+
std::vector<at::Tensor> outputs;
68+
69+
std::vector<std::future<at::Tensor>> futures;
70+
for (const auto i : c10::irange(nthreads)) {
71+
(void)i;
72+
futures.push_back(std::async(std::launch::async, [&model]() {
73+
auto input = torch::ones({10, 10, 10});
74+
for (const auto j : c10::irange(100)) {
75+
(void)j;
76+
model({input.alias()}).toTensor();
77+
}
78+
auto result = model({input.alias()}).toTensor();
79+
return result;
80+
}));
81+
}
82+
for (const auto i : c10::irange(nthreads)) {
83+
outputs.push_back(futures[i].get());
84+
}
85+
86+
// Generate reference
87+
auto ref_output = ref_model.forward({input.alias()}).toTensor();
88+
89+
// Compare all to reference
90+
for (const auto i : c10::irange(nthreads)) {
91+
ASSERT_TRUE(ref_output.equal(outputs[i]));
92+
}
93+
}
94+
95+
int main(int argc, char* argv[]) {
96+
::testing::InitGoogleTest(&argc, argv);
97+
char tempeh[256];
98+
getcwd(tempeh, 256);
99+
std::cout << "Current working directory: " << tempeh << std::endl;
100+
int rc = RUN_ALL_TESTS();
101+
char tmp[256];
102+
getcwd(tmp, 256);
103+
std::cout << "Current working directory: " << tmp << std::endl;
104+
return rc;
105+
}

0 commit comments

Comments
 (0)