Merge from Ctuning

README.md

@@ -45,7 +45,8 @@ that can be chained into more complex automation workflows [using simple JSON or
 
 *Note that MLCommons CM is a collaborative engineering effort to gradually improve portability and functionality
 across continuously changing models, data sets, software and hardware based on your feedback -
-please check this [installation guide](installation.md), report  encountered issues [here](https://github.com/mlcommons/ck/issues) 
+please check this [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md), 
+report  encountered issues [here](https://github.com/mlcommons/ck/issues) 
 and contact us via [public Discord Server](https://discord.gg/JjWNWXKxwT) to help this community effort!*
 
 

cm-mlops/script/compile-model-for.qaic/_cm.json

@@ -305,12 +305,12 @@
     },
     "bert-99.9,server": {
       "env": {
-        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2"
       }
     },
     "bert-99.9,server,nsp.14": {
       "env": {
-        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2"
       }
     },
     "tf": {

cm-mlops/script/generate-mlperf-inference-submission/customize.py

@@ -59,7 +59,7 @@ def generate_submission(i):
         system_meta['division'] = env['CM_MLPERF_SUBMISSION_DIVISION']
 
     if 'CM_MLPERF_SUBMISSION_CATEGORY' in env:
-        system_meta['system_type'] = env['CM_MLPERF_SUBMISSION_CATEGORY']
+        system_meta['system_type'] = env['CM_MLPERF_SUBMISSION_CATEGORY'].replace("-", ",")
 
     duplicate= (env.get('CM_MLPERF_DUPLICATE_SCENARIO_RESULTS', 'no') in ["yes", "True"])
 

cm-mlops/script/gui/install/linux.md

@@ -0,0 +1,10 @@
+```bash
+sudo apt update && sudo apt upgrade
+sudo apt install python3 python3-pip python3-venv git wget curl
+```
+
+*Note that you must set up virtual env on Ubuntu 23+ before using any Python project including CM:*
+```bash
+python3 -m venv cm
+source cm/bin/activate
+```

cm-mlops/script/gui/install/macos.md

@@ -0,0 +1,22 @@
+*Note that CM currently does not work with Python installed from the Apple Store.
+ Please install Python via brew as described below.*
+
+If `brew` package manager is not installed, please install it as follows (see details [here](https://brew.sh/)):
+```bash
+/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+```
+
+Don't forget to add brew to PATH environment as described in the end.
+
+Then install python, pip, git and wget:
+
+```bash
+brew install python3 git wget curl
+
+python3 -m pip install cmind
+```
+
+*Sometimes python does not add `cm` and `cmr` binaries to the `PATH` environment variable.
+ You may need to find these files and add their path to `PATH` variable.
+ We plan to simplify this installation in the future.*
+

cm-mlops/script/gui/install/redhat.md

@@ -0,0 +1,7 @@
+*We have successfully tested CM on Red Hat 9 and CentOS 8:*
+
+```bash
+sudo dnf update
+sudo dnf install python3 python-pip git wget curl
+
+```

cm-mlops/script/gui/install/windows.md

@@ -0,0 +1,13 @@
+* Configure Windows 10+ to support long paths as described 
+  [here](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later).
+* Download and install Git from [git-for-windows.github.io](https://git-for-windows.github.io).
+  * Configure Git to accept long file names: `git config --system core.longpaths true`
+* Download and install Python 3+ from [www.python.org/downloads/windows](https://www.python.org/downloads/windows).
+  * Don't forget to select option to add Python binaries to PATH environment!
+  * Configure Windows to accept long fie names during Python installation!
+
+*Note that we [have reports](https://github.com/mlcommons/ck/issues/844) 
+ that CM does not work when Python was first installed from the Microsoft Store.
+ If CM fails to run, you can find a fix [here](https://stackoverflow.com/questions/57485491/python-python3-executes-in-command-prompt-but-does-not-run-correctly)*.
+
+*We plan to provide a self-sustained package in the future to simplify CM installation on Windows.*

cm-mlops/script/gui/playground.py

@@ -79,6 +79,7 @@ def main():
     style_action_apps='font-style:italic;font-weight:bold;color:#ffffff' if action=='apps' else ''
     style_action_reports='font-style:italic;font-weight:bold;color:#ffffff' if action=='reports' else ''
     style_action_beta='font-style:italic;font-weight:bold;color:#ffffff' if action=='beta' else ''
+    style_action_install='font-style:italic;font-weight:bold;color:#ffffff' if action=='install' else ''
 
     st.write('''
         <center>
@@ -93,6 +94,7 @@ def main():
         <a target="_self" href="?action=scripts&tags=modular,app"><button style="{}">Modular apps</button></a><br>
         <a target="_self" href="https://discord.gg/JjWNWXKxwT"><button>Discord</button></a>
         <a target="_self" href="https://github.com/mlcommons/ck"><button>GitHub</button></a>
+        <a target="_self" href="?action=install"><button style="{}">Install</button></a>
         </center>
         '''.format(
                    style_action_scripts,
@@ -103,7 +105,8 @@ def main():
                    style_action_contributors,
                    style_action_reports,
                    style_action_beta,
-                   style_action_apps
+                   style_action_apps,
+                   style_action_install
                    ),
         unsafe_allow_html=True
     )
@@ -141,6 +144,9 @@ def main():
     elif action == 'beta':
         from playground_beta import page
         r = page(st, params)
+    elif action == 'install' or action == 'setup':
+        from playground_install import page
+        r = page(st, params, {})
 
     if r['return']>0:
         st.markdown('**CM error:** {} . Please report [here](https://github.com/mlcommons/ck/issues)'.format(r['error']))

cm-mlops/script/gui/playground_howtorun.py

@@ -38,12 +38,12 @@ def page(st, params, action = ''):
          <small>
          This interface will help you generate a command line or Python API 
          to run modular benchmarks composed from 
-         <a href="{}">automation recipes (CM scripts)</a>
-         across different models, data sets, software and hardware.
+         <a href="{}">automation recipes (CM scripts)</a>.
          Note that this is a <a href="https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md">collaborative engineering effort</a> 
-         - please report issues and suggestions 
+         to make sure that they work across all possible versions and configurations of models, data sets, software and hardware
+         - please report encountered issues and provide feedback
          <a href="https://github.com/mlcommons/ck/issues">here</a>
-         and get in touch via <a href="https://discord.gg/JjWNWXKxwT">Discord</a> for more details.
+         and get in touch via <a href="https://discord.gg/JjWNWXKxwT">Discord</a>!
          </small>
          </i>
           <br>

cm-mlops/script/gui/playground_install.py

@@ -0,0 +1,141 @@
+# Developer(s): Grigori Fursin
+
+import cmind
+import os
+import datetime
+import misc
+
+def page(st, params, extra):
+
+    end_html = ''
+
+    url_prefix = st.config.get_option('server.baseUrlPath')+'/'
+
+    if not extra.get('skip_header', False):
+        st.markdown('---')
+        st.markdown('**Install [MLCommons Collective Mind automation framework](https://github.com/mlcommons/ck):**')
+
+
+
+    md = ''
+
+    ###################################################################
+    # Select OS
+    choices = [('Ubuntu, Debian and similar Linux', 'linux'),
+               ('Red Hat and CentOS', 'redhat'),
+               ('MacOS', 'macos'),
+               ('Windows', 'windows')]
+
+    host_os_selection = 0
+
+    if extra.get('run_on_windows', False):
+        host_os_selection = 3
+
+    host_os = st.selectbox('Select your host OS:', 
+                           range(len(choices)),
+                           format_func = lambda x: choices[x][0], 
+                           index = host_os_selection, 
+                           key = 'install_select_host_os')
+
+    host_os_index = choices[host_os][1]
+
+
+    cur_script_file = __file__
+    cur_script_path = os.path.dirname(cur_script_file)
+
+
+    notes = os.path.join(cur_script_path, 'install', host_os_index+'.md')
+
+    if os.path.isfile(notes):
+        r = cmind.utils.load_txt(notes)
+        if r['return']>0: return r
+        s = r['string']
+        if s != '':
+            show = st.toggle('Show system dependencies?', value = True)
+            if show:
+                md += s
+
+
+    need_user = ''
+    python = 'python3'
+    if host_os_index == 'redhat':
+        need_user = ' --user'
+    elif host_os_index == 'windows':
+        python = 'python'
+
+
+    ###################################################################
+    # Select repository
+
+    choices = [('Stable Git version from GitHub: mlcommons@ck', 'stable'),
+               ('Dev Git version from GitHub: ctuning@mlcommons-ck', 'ctuning'),
+               ('Small and stable ZIP from Zenodo: 20240223', 'zenodo')]
+
+    repo = st.selectbox('Select repository with [automation recipes (CM scripts)](https://access.cknowledge.org/playground/?action=scripts):',
+                         range(len(choices)),
+                         format_func = lambda x: choices[x][0], 
+                         index=0, 
+                         key='select_repo')
+
+    repo_index = choices[repo][1]
+
+
+    # Add stable repo from Zenodo
+    if repo_index == 'ctuning':
+        cm_repo = 'ctuning@mlcommons-ck'
+    elif repo_index == 'zenodo':
+        cm_repo = '--url=https://zenodo.org/records/10679842/files/cm-mlops-repo-20240219.zip'
+    else:
+        cm_repo = 'mlcommons@ck'
+
+    x =  '{} -m pip install cmind -U{}\n\n'.format(python, need_user)
+    x += 'cm pull repo {}\n\n'.format(cm_repo)
+
+    clean_cm_cache = st.toggle('Clean CM cache', value=True, key = 'install_clean_cm_cache')
+
+    cm_clean_cache = 'cm rm cache -f\n\n' if clean_cm_cache else ''
+
+    x += cm_clean_cache
+
+
+
+    python_venv_name=params.get('@adr.python.name', '')
+    python_ver_min=params.get('@adr.python.version_min', '')
+    python_ver=params.get('@adr.python.version', '')
+
+    if python_venv_name == '':
+         use_python_venv = st.toggle('Use Python Virtual Environment for CM scripts?', value = False)
+         if use_python_venv:
+             python_venv_name = st.text_input('Enter some CM python venv name for your project:', value = "mlperf-v4.0")
+
+             if python_ver_min == '':
+                 python_ver_min = st.text_input('[Optional] Specify min version such as 3.8:')
+
+    y = ''
+    if python_venv_name!='':# or python_ver!='' or python_ver_min!='':
+        y = 'cm run script "get sys-utils-cm"\n'
+
+        if python_venv_name!='':
+            y+='cm run script "install python-venv" --name='+str(python_venv_name)
+        else:
+            y+='cm run script "get python"'
+
+        if python_ver!='':
+            y+=' --version='+str(python_ver)
+
+        if python_ver_min!='':
+            y+=' --version_min='+str(python_ver_min)
+
+    if y!='':
+        x+=y
+
+
+    md += '```bash\n{}\n```\n'.format(x)
+
+    st.markdown('---')
+    st.markdown(md)
+    st.markdown('*Check [more CM installation notes at GitHub](https://github.com/mlcommons/ck/blob/master/docs/installation.md)*.')
+
+
+
+    return {'return':0, 'end_html':end_html}

cm-mlops/script/gui/playground_reproduce.py

@@ -48,14 +48,16 @@ def page(st, params, action = ''):
     x = '''
          <i>
          <small>
-         This interface will help you find the <a href="{}">modular benchmarks' settings</a> 
-         that <a href="{}">the community</a> have managed to successfully validate 
+         [Under development] This is a new project to reproduce <a href="{}">modular benchmarks</a> 
          across different models, data sets, software and hardware 
-         based on the <a href="https://cTuning.org/ae">ACM/cTuning reproducibility methodology and badges</a>.
+         via <a href="{}">open challenges</a>
+         based on the <a href="https://cTuning.org/ae">ACM/cTuning reproducibility methodology and badges</a>
+         and <a href="https://sites.google.com/g.harvard.edu/mlperf-bench-hpca24/home">automatically compose 
+         High-Performance and Cost-Efficient AI Systems with MLCommons' Collective Mind and MLPerf</a>.
          Note that this is a <a href="https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md">collaborative engineering effort</a> 
-         - please report issues and suggestions 
+         - please report encountered issues and provide feedback
          <a href="https://github.com/mlcommons/ck/issues">here</a>
-         and get in touch via <a href="https://discord.gg/JjWNWXKxwT">Discord</a> for more details.
+         and get in touch via <a href="https://discord.gg/JjWNWXKxwT">Discord</a>!
          </small>
          </i>
           <br>