Skip to content
This repository was archived by the owner on Dec 11, 2024. It is now read-only.

Commit 81b22b3

Browse files
anandhu-enganandhu-eng
authored
Support MLPer inference rgat reference implementation (#50)
1 parent c8bf646 commit 81b22b3

File tree

16 files changed

+141
-39
lines changed

16 files changed

+141
-39
lines changed

‎script/app-mlperf-inference-mlcommons-python/_cm.yaml

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -482,13 +482,12 @@ deps:
482482
## RGAT
483483
- tags: get,ml-model,rgat
484484
names:
485-
- ml-model
486485
- rgat-model
487486
enable_if_env:
488487
CM_MODEL:
489488
- rgat
490489
skip_if_env:
491-
RGAT_CHECKPOINT_PATH:
490+
CM_ML_MODEL_RGAT_CHECKPOINT_PATH:
492491
- 'on'
493492

494493
########################################################################
@@ -620,6 +619,9 @@ deps:
620619
enable_if_env:
621620
CM_MODEL:
622621
- rgat
622+
skip_if_env:
623+
CM_DATASET_IGBH_PATH:
624+
- "on"
623625

624626
########################################################################
625627
# Install MLPerf inference dependencies
@@ -1224,27 +1226,45 @@ variations:
12241226
group: models
12251227
env:
12261228
CM_MODEL: rgat
1229+
adr:
1230+
pytorch:
1231+
version: 2.1.0
12271232
deps:
12281233
- tags: get,generic-python-lib,_package.colorama
12291234
- tags: get,generic-python-lib,_package.tqdm
12301235
- tags: get,generic-python-lib,_package.requests
12311236
- tags: get,generic-python-lib,_package.torchdata
1232-
- tags: get,generic-python-lib,_package.torch-geometric
1233-
- tags: get,generic-python-lib,_package.torch-scatter
1234-
- tags: get,generic-python-lib,_package.torch-sparse
1237+
version: 0.7.0
1238+
- tags: get,generic-python-lib,_package.torchvision
1239+
version: 0.16.0
12351240
- tags: get,generic-python-lib,_package.pybind11
12361241
- tags: get,generic-python-lib,_package.PyYAML
1242+
- tags: get,generic-python-lib,_package.numpy
1243+
version: 1.26.4
12371244
- tags: get,generic-python-lib,_package.pydantic
12381245
- tags: get,generic-python-lib,_package.igb,_url.git+https://github.com/IllinoisGraphBenchmark/IGB-Datasets.git
1239-
- tags: get,generic-python-lib,_package.dgl,_find_links_url.https://data.dgl.ai/wheels/torch-2.1/repo.html
1240-
enable_if_env:
1241-
CM_MLPERF_DEVICE:
1242-
- cpu
1246+
1247+
rgat,cuda:
1248+
deps:
12431249
- tags: get,generic-python-lib,_package.dgl,_find_links_url.https://data.dgl.ai/wheels/torch-2.1/cu121/repo.html
1244-
enable_if_env:
1245-
CM_MLPERF_DEVICE:
1246-
- gpu
1247-
1250+
- tags: get,generic-python-lib,_package.torch-scatter
1251+
- tags: get,generic-python-lib,_package.torch-sparse
1252+
- tags: get,generic-python-lib,_package.torch-geometric
1253+
env:
1254+
CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<CM_TORCH_VERSION>>>.html"
1255+
1256+
rgat,cpu:
1257+
deps:
1258+
- tags: get,generic-python-lib,_package.torch-geometric
1259+
env:
1260+
CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<CM_TORCH_VERSION>>>+cpu.html"
1261+
- tags: get,generic-python-lib,_package.torch-scatter
1262+
env:
1263+
CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<CM_TORCH_VERSION>>>+cpu.html"
1264+
- tags: get,generic-python-lib,_package.torch-sparse
1265+
env:
1266+
CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<CM_TORCH_VERSION>>>+cpu.html"
1267+
- tags: get,generic-python-lib,_package.dgl,_find_links_url.https://data.dgl.ai/wheels/torch-2.1/repo.html
12481268

12491269
# Target devices
12501270
cpu:

‎script/app-mlperf-inference-mlcommons-python/customize.py

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,12 @@ def preprocess(i):
115115
scenario_extra_options = ''
116116

117117
NUM_THREADS = env['CM_NUM_THREADS']
118-
if int(NUM_THREADS) > 2 and env['CM_MLPERF_DEVICE'] == "gpu":
118+
if int(
119+
NUM_THREADS) > 2 and env['CM_MLPERF_DEVICE'] == "gpu" and env['CM_MODEL'] != "rgat":
119120
NUM_THREADS = "2" # Don't use more than 2 threads when run on GPU
120121

121-
if env['CM_MODEL'] in ['resnet50', 'retinanet', 'stable-diffusion-xl']:
122+
if env['CM_MODEL'] in ['resnet50', 'retinanet',
123+
'stable-diffusion-xl', 'rgat']:
122124
scenario_extra_options += " --threads " + NUM_THREADS
123125

124126
ml_model_name = env['CM_MODEL']
@@ -485,15 +487,16 @@ def get_run_cmd_reference(
485487
# have to add the condition for running in debug mode or real run mode
486488
cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \
487489
" --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \
488-
" --dataset-path " + env['CM_IGBH_DATASET_PATH'] + \
489-
" --device " + device.replace("cuda", "cuda:0") + \
490+
" --dataset-path " + env['CM_DATASET_IGBH_PATH'] + \
491+
" --device " + device.replace("cuda", "gpu") + \
490492
env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
491493
scenario_extra_options + mode_extra_options + \
492494
" --output " + env['CM_MLPERF_OUTPUT_DIR'] + \
493495
' --dtype ' + dtype_rgat + \
494-
" --model-path " + env['RGAT_CHECKPOINT_PATH'] + \
495-
" --mlperf_conf " + \
496-
os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "mlperf.conf")
496+
" --model-path " + env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH']
497+
498+
if env.get('CM_ACTIVATE_RGAT_IN_MEMORY', '') == "yes":
499+
cmd += " --in-memory "
497500

498501
if env.get('CM_NETWORK_LOADGEN', '') in ["lon", "sut"]:
499502
cmd = cmd + " " + "--network " + env['CM_NETWORK_LOADGEN']

‎script/app-mlperf-inference/_cm.yaml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,20 @@ variations:
767767
env:
768768
CM_MODEL:
769769
rgat
770+
posthook_deps:
771+
- enable_if_env:
772+
CM_MLPERF_LOADGEN_MODE:
773+
- accuracy
774+
- all
775+
CM_MLPERF_ACCURACY_RESULTS_DIR:
776+
- 'on'
777+
skip_if_env:
778+
CM_MLPERF_IMPLEMENTATION:
779+
- nvidia
780+
names:
781+
- mlperf-accuracy-script
782+
- 3d-unet-accuracy-script
783+
tags: run,accuracy,mlperf,_igbh
770784

771785
sdxl:
772786
group:
@@ -1645,6 +1659,25 @@ variations:
16451659
CM_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp38-cp38-linux_x86_64.whl'
16461660
CM_MLPERF_INFERENCE_VERSION: '4.1'
16471661

1662+
r5.0-dev_default:
1663+
group:
1664+
reproducibility
1665+
add_deps_recursive:
1666+
nvidia-inference-common-code:
1667+
version: r4.1
1668+
tags: _mlcommons
1669+
nvidia-inference-server:
1670+
version: r4.1
1671+
tags: _mlcommons
1672+
intel-harness:
1673+
tags: _v4.1
1674+
default_env:
1675+
CM_SKIP_SYS_UTILS: 'yes'
1676+
CM_REGENERATE_MEASURE_FILES: 'yes'
1677+
env:
1678+
CM_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp38-cp38-linux_x86_64.whl'
1679+
1680+
16481681
invalid_variation_combinations:
16491682
-
16501683
- retinanet
@@ -1768,6 +1801,8 @@ docker:
17681801
- "${{ CM_NVIDIA_LLAMA_DATASET_FILE_PATH }}:${{ CM_NVIDIA_LLAMA_DATASET_FILE_PATH }}"
17691802
- "${{ SDXL_CHECKPOINT_PATH }}:${{ SDXL_CHECKPOINT_PATH }}"
17701803
- "${{ CM_DATASET_KITS19_PREPROCESSED_PATH }}:${{ CM_DATASET_KITS19_PREPROCESSED_PATH }}"
1804+
- "${{ CM_DATASET_IGBH_PATH }}:${{ CM_DATASET_IGBH_PATH }}"
1805+
- "${{ CM_ML_MODEL_RGAT_CHECKPOINT_PATH }}:${{ CM_ML_MODEL_RGAT_CHECKPOINT_PATH }}"
17711806
skip_run_cmd: 'no'
17721807
shm_size: '32gb'
17731808
interactive: True

‎script/get-cudnn/_cm.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ default_env:
1919

2020
deps:
2121
- tags: detect,os
22+
- tags: detect,sudo
2223
- names:
2324
- cuda
2425
skip_if_env:

‎script/get-dataset-mlperf-inference-igbh/_cm.yaml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ tags:
1111
- inference
1212
uid: 824e61316c074253
1313
new_env_keys:
14-
- CM_IGBH_DATASET_PATH
14+
- CM_DATASET_IGBH_PATH
15+
- CM_DATASET_IGBH_SIZE
1516
input_mapping:
1617
out_path: CM_IGBH_DATASET_OUT_PATH
1718
deps:
@@ -21,6 +22,9 @@ deps:
2122
- tags: get,python
2223
names:
2324
- get-python
25+
- tags: get,generic-python-lib,_package.igb,_url.git+https://github.com/anandhu-eng/IGB-Datasets.git
26+
- tags: get,generic-python-lib,_package.colorama
27+
- tags: get,generic-python-lib,_package.tqdm
2428
prehook_deps:
2529
#paper
2630
- env:
@@ -359,13 +363,13 @@ variations:
359363
default: true
360364
group: dataset-type
361365
env:
362-
CM_IGBH_DATASET_TYPE: debug
363-
CM_IGBH_DATASET_SIZE: tiny
366+
CM_DATASET_IGBH_TYPE: debug
367+
CM_DATASET_IGBH_SIZE: tiny
364368
full:
365369
group: dataset-type
366370
env:
367-
CM_IGBH_DATASET_TYPE: full
368-
CM_IGBH_DATASET_SIZE: full
371+
CM_DATASET_IGBH_TYPE: debug
372+
CM_DATASET_IGBH_SIZE: tiny
369373
glt:
370374
env:
371375
CM_IGBH_GRAPH_COMPRESS: yes

‎script/get-dataset-mlperf-inference-igbh/customize.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,18 @@ def preprocess(i):
2727
x_sep = " && "
2828

2929
# download the model
30-
if env['CM_IGBH_DATASET_TYPE'] == "debug":
30+
if env['CM_DATASET_IGBH_TYPE'] == "debug":
3131
run_cmd += x_sep + env['CM_PYTHON_BIN_WITH_PATH'] + \
3232
f" tools/download_igbh_test.py --target-path {download_loc} "
3333

3434
# split seeds
3535
run_cmd += x_sep + \
36-
f"{env['CM_PYTHON_BIN_WITH_PATH']} tools/split_seeds.py --path {download_loc} --dataset_size {env['CM_IGBH_DATASET_SIZE']}"
36+
f"{env['CM_PYTHON_BIN_WITH_PATH']} tools/split_seeds.py --path {download_loc} --dataset_size {env['CM_DATASET_IGBH_SIZE']}"
3737

3838
# compress graph(for glt implementation)
3939
if env.get('CM_IGBH_GRAPH_COMPRESS', '') == "yes":
4040
run_cmd += x_sep + \
41-
f"{env['CM_PYTHON_BIN_WITH_PATH']} tools/compress_graph.py --path {download_loc} --dataset_size {env['CM_IGBH_DATASET_SIZE']} --layout {env['CM_IGBH_GRAPH_COMPRESS_LAYOUT']}"
41+
f"{env['CM_PYTHON_BIN_WITH_PATH']} tools/compress_graph.py --path {download_loc} --dataset_size {env['CM_DATASET_IGBH_SIZE']} --layout {env['CM_IGBH_GRAPH_COMPRESS_LAYOUT']}"
4242

4343
env['CM_RUN_CMD'] = run_cmd
4444

@@ -49,10 +49,10 @@ def postprocess(i):
4949

5050
env = i['env']
5151

52-
env['CM_IGBH_DATASET_PATH'] = env.get(
52+
env['CM_DATASET_IGBH_PATH'] = env.get(
5353
'CM_IGBH_DATASET_OUT_PATH', os.getcwd())
5454

5555
print(
56-
f"Path to the IGBH dataset: {os.path.join(env['CM_IGBH_DATASET_PATH'], env['CM_IGBH_DATASET_SIZE'])}")
56+
f"Path to the IGBH dataset: {os.path.join(env['CM_DATASET_IGBH_PATH'], env['CM_DATASET_IGBH_SIZE'])}")
5757

5858
return {'return': 0}

‎script/get-ml-model-rgat/_cm.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ input_mapping:
1212
to: CM_DOWNLOAD_PATH
1313
new_env_keys:
1414
- CM_ML_MODEL_*
15-
- RGAT_CHECKPOINT_PATH
15+
- CM_ML_MODEL_RGAT_CHECKPOINT_PATH
1616
prehook_deps:
1717
- enable_if_env:
1818
CM_DOWNLOAD_TOOL:

‎script/get-ml-model-rgat/customize.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@ def postprocess(i):
1919

2020
env = i['env']
2121

22-
if env.get('RGAT_CHECKPOINT_PATH', '') == '':
23-
env['RGAT_CHECKPOINT_PATH'] = os.path.join(
22+
if env.get('CM_ML_MODEL_RGAT_CHECKPOINT_PATH', '') == '':
23+
env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = os.path.join(
2424
env['CM_ML_MODEL_PATH'], "RGAT.pt")
2525
elif env.get('CM_ML_MODEL_PATH', '') == '':
26-
env['CM_ML_MODEL_PATH'] = env['RGAT_CHECKPOINT_PATH']
26+
env['CM_ML_MODEL_PATH'] = env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH']
2727

28-
env['CM_GET_DEPENDENT_CACHED_PATH'] = env['RGAT_CHECKPOINT_PATH']
28+
env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH']
2929

3030
return {'return': 0}

‎script/get-mlperf-inference-src/_cm.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,18 @@ versions:
166166
env:
167167
CM_MLPERF_LAST_RELEASE: v3.1
168168
CM_TMP_GIT_CHECKOUT: ''
169+
r4.0:
170+
adr:
171+
inference-git-repo:
172+
tags: _tag.v4.0
173+
env:
174+
CM_MLPERF_LAST_RELEASE: v4.0
175+
r4.1:
176+
adr:
177+
inference-git-repo:
178+
tags: _tag.v4.1
179+
env:
180+
CM_MLPERF_LAST_RELEASE: v4.1
169181
tvm:
170182
env:
171183
CM_MLPERF_LAST_RELEASE: v3.1

‎script/get-mlperf-inference-src/customize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def preprocess(i):
5454
env["CM_GIT_URL"] = "https://github.com/mlcommons/inference"
5555

5656
if env.get("CM_MLPERF_LAST_RELEASE", '') == '':
57-
env["CM_MLPERF_LAST_RELEASE"] = "v4.1"
57+
env["CM_MLPERF_LAST_RELEASE"] = "v5.0"
5858

5959
if 'CM_GIT_DEPTH' not in env:
6060
env['CM_GIT_DEPTH'] = ''

‎script/process-mlperf-accuracy/_cm.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,3 +261,7 @@ variations:
261261
env:
262262
CM_DATASET: terabyte
263263
group: dataset
264+
igbh:
265+
env:
266+
CM_DATASET: igbh
267+
group: dataset

‎script/process-mlperf-accuracy/customize.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,16 @@ def preprocess(i):
171171
" --dtype " + env.get('CM_ACCURACY_DTYPE',
172172
"float32") + " > '" + out_file + "'"
173173

174+
elif dataset == "igbh":
175+
if env.get('CM_DATASET_IGBH_SIZE', '') == '':
176+
if env.get('CM_MLPERF_SUBMISSION_GENERATION_STYLE',
177+
'') == "full":
178+
env['CM_DATASET_IGBH_SIZE'] = "full"
179+
else:
180+
env['CM_DATASET_IGBH_SIZE'] = "tiny"
181+
CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join(
182+
result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' > '" + out_file + "'"
183+
174184
else:
175185
return {'return': 1, 'error': 'Unsupported dataset'}
176186

‎script/run-mlperf-inference-app/_cm.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,19 @@ variations:
360360
mlperf-inference-nvidia-scratch-space:
361361
tags: _version.r4_1
362362
group: benchmark-version
363+
364+
r5.0-dev:
365+
env:
366+
CM_MLPERF_INFERENCE_VERSION: '5.0-dev'
367+
CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r5.0-dev_default
368+
group: benchmark-version
369+
adr:
370+
get-mlperf-inference-results-dir:
371+
tags: _version.r5.0-dev
372+
get-mlperf-inference-submission-dir:
373+
tags: _version.r5.0-dev
374+
mlperf-inference-nvidia-scratch-space:
375+
tags: _version.r5.0-dev
363376

364377
short:
365378
add_deps_recursive:

‎script/test-cm-core/src/tutorials/test_tutorial_retinanet.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
'name': 'mlperf'})
3131
checks.check_return(r)
3232

33-
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr':
33+
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr':
3434
{'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community',
3535
'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline',
3636
'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'})

‎script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'})
1919
checks.check_return(r)
2020

21-
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr':
21+
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr':
2222
{'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {
2323
'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}},
2424
'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx',

‎script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'})
2121
checks.check_return(r)
2222

23-
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr':
23+
r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr':
2424
{'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community',
2525
'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline',
2626
'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'})

0 commit comments

Comments
 (0)