refactor: add commandTransform utility (#2736)

.gitignore

@@ -5,3 +5,5 @@ coverage
 .vscode
 package-lock.json
 .nyc_output
+rawScripts
+lib/scripts

.npmignore

@@ -11,3 +11,10 @@ support
 .gitignore
 *.md
 commitlint.config.js
+rawScripts
+commandTransform.js
+docker-compose.yml
+generateRawScripts.js
+.nyc_output
+.mocharc.json
+lib/commands/*.js

commandTransform.js

@@ -0,0 +1,67 @@
+'use strict';
+const path = require('path');
+const fs = require('fs');
+const { argv } = require('process');
+
+const readFile = fs.promises.readFile;
+const writeFile = fs.promises.writeFile;
+const readdir = fs.promises.readdir;
+
+const loadScripts = async (readDir, writeDir) => {
+  const normalizedDir = path.normalize(readDir);
+
+  const files = await readdir(normalizedDir);
+
+  const luaFiles = files.filter(file => path.extname(file) === '.lua');
+  const writeFilenamePath = path.normalize(writeDir);
+
+  if (!fs.existsSync(writeFilenamePath)) {
+    fs.mkdirSync(writeFilenamePath);
+  }
+
+  let indexContent = "'use strict';\nmodule.exports = {\n";
+
+  if (luaFiles.length === 0) {
+    /**
+     * To prevent unclarified runtime error "updateDelayset is not a function
+     * @see https://github.com/OptimalBits/bull/issues/920
+     */
+    throw new Error('No .lua files found!');
+  }
+
+  for (let i = 0; i < luaFiles.length; i++) {
+    const completedFilename = path.join(normalizedDir, luaFiles[i]);
+    const longName = path.basename(luaFiles[i], '.lua');
+    indexContent += `  ["${longName}"]: require('./${longName}'),\n`;
+
+    await loadCommand(completedFilename, longName, writeFilenamePath);
+  }
+  indexContent += `}\n`;
+
+  await writeFile(path.join(writeFilenamePath, 'index.js'), indexContent);
+};
+
+const loadCommand = async (filename, longName, writeFilenamePath) => {
+  const filenamePath = path.resolve(filename);
+
+  const content = (await readFile(filenamePath)).toString();
+
+  const [name, num] = longName.split('-');
+  const numberOfKeys = num && parseInt(num, 10);
+
+  const newContent = `'use strict';
+const content = \`${content}\`;
+module.exports = {
+  name: '${name}',
+  content,${
+    numberOfKeys
+      ? `
+  keys: ${numberOfKeys},`
+      : ''
+  }
+};
+`;
+  await writeFile(path.join(writeFilenamePath, longName + '.js'), newContent);
+};
+
+loadScripts(argv[2], argv[3]);

generateRawScripts.js

@@ -0,0 +1,45 @@
+'use strict';
+const { ScriptLoader } = require('./lib/commands');
+const path = require('path');
+const fs = require('fs');
+const { promisify } = require('util');
+
+const writeFile = promisify(fs.writeFile);
+
+class RawScriptLoader extends ScriptLoader {
+  /**
+   * Transpile lua scripts in one file, specifying an specific directory to be saved
+   * @param pathname - the path to the directory containing the scripts
+   * @param writeDir - the path to the directory where scripts will be saved
+   */
+  async transpileScripts(pathname, writeDir) {
+    const writeFilenamePath = path.normalize(writeDir);
+
+    if (!fs.existsSync(writeFilenamePath)) {
+      fs.mkdirSync(writeFilenamePath);
+    }
+
+    const paths = new Set();
+    if (!paths.has(pathname)) {
+      paths.add(pathname);
+      const scripts = await this.loadScripts(pathname);
+      for (const command of scripts) {
+        const {
+          name,
+          options: { numberOfKeys, lua },
+        } = command;
+        await writeFile(
+          path.join(writeFilenamePath, `${name}-${numberOfKeys}.lua`),
+          lua,
+        );
+      }
+    }
+  }
+}
+
+const scriptLoader = new RawScriptLoader();
+
+scriptLoader.transpileScripts(
+  path.join(__dirname, './lib/commands'),
+  path.join(__dirname, './rawScripts'),
+);

lib/commands/index.js

@@ -1,54 +1,8 @@
-/**
- * Load redis lua scripts.
- * The name of the script must have the following format:
- *
- * cmdName-numKeys.lua
- *
- * cmdName must be in camel case format.
- *
- * For example:
- * moveToFinish-3.lua
- *
- */
 'use strict';
+const { ScriptLoader } = require('./script-loader');
 
-const fsAsync = require('fs').promises;
-const path = require('path');
+const scriptLoader = new ScriptLoader();
 
-module.exports = (function() {
-  let scripts;
-
-  return async function(client) {
-    scripts = await (scripts || loadScripts());
-
-    return scripts.map(({ name, options }) =>
-      client.defineCommand(name, options)
-    );
-  };
-})();
-
-async function loadScripts() {
-  const scriptsDir = await fsAsync.readdir(__dirname);
-  const luaFiles = scriptsDir.filter(file => path.extname(file) === '.lua');
-  if (luaFiles.length === 0) {
-    /**
-     * To prevent unclarified runtime error "updateDelayset is not a function
-     * @see https://github.com/OptimalBits/bull/issues/920
-     */
-    throw new Error('No .lua files found!');
-  }
-  return Promise.all(
-    luaFiles.map(async file => {
-      const lua = await fsAsync.readFile(path.join(__dirname, file));
-      const longName = path.basename(file, '.lua');
-
-      return {
-        name: longName.split('-')[0],
-        options: {
-          numberOfKeys: parseInt(longName.split('-')[1]),
-          lua: lua.toString()
-        }
-      };
-    })
-  );
+module.exports = {
+  ScriptLoader, scriptLoader
 }

lib/commands/moveStalledJobsToWait-7.lua

@@ -75,7 +75,7 @@ if(#stalling > 0) then
         if(stalledCount > MAX_STALLED_JOB_COUNT) then
           rcall("ZADD", KEYS[4], ARGV[3], jobId)
           rcall("HSET", jobKey, "failedReason", "job stalled more than allowable limit")
-          rcall("PUBLISH", KEYS[4],  "{\"jobId\":\"" .. jobId .. "\", \"val\": \"job stalled more than maxStalledCount\"}")
+          rcall("PUBLISH", KEYS[4],  '{"jobId":"' .. jobId .. '", "val": "job stalled more than maxStalledCount"}')
           table.insert(failed, jobId)
         else
           -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.

lib/commands/obliterate-2.lua

@@ -13,7 +13,7 @@
 -- keys, consider that this call may be slow for very large queues.
 -- The queue needs to be "paused" or it will return an error
 -- If the queue has currently active jobs then the script by default will return error,
--- however this behaviour can be overrided using the `force` option.
+-- however this behaviour can be overrided using the 'force' option.
 local maxCount = tonumber(ARGV[1])
 local baseKey = KEYS[2]