adding support for optimizing all plugin resources. Still a bit experimental.

Author: jrburke

build/example.build.js

@@ -124,6 +124,14 @@
     //together.
     skipModuleInsertion: false,
 
+    //If it is not a one file optimization, scan through all .js files in the
+    //output directory for any plugin resource dependencies, and if the plugin
+    //supports optimizing them as separate files, optimize them. Can be a
+    //slower optimization. Only use if there are some plugins that use things
+    //like XMLHttpRequest that do not work across domains, but the built code
+    //will be placed on another domain.
+    optimizeAllPluginResources: false,
+
     //List the modules that will be optimized. All their immediate and deep
     //dependencies will be included in the module's file when the build is
     //done. If that module or any of its dependencies includes i18n bundles,

build/jslib/build.js

@@ -22,7 +22,8 @@ function (lang,   logger,   file,          parse,    optimize,   pragma,
             optimize: "uglify",
             optimizeCss: "standard.keepLines",
             inlineText: true,
-            isBuild: true
+            isBuild: true,
+            optimizeAllPluginResources: false
         };
 
     /**
@@ -47,6 +48,13 @@ function (lang,   logger,   file,          parse,    optimize,   pragma,
         return dirName;
     }
 
+    //Method used by plugin writeFile calls, defined up here to avoid
+    //jslint warning about "making a function in a loop".
+    function writeFile(name, contents) {
+        logger.trace('Saving plugin-optimized file: ' + name);
+        file.saveUtf8File(name, contents);
+    }
+
     /**
      * Main API entry point into the build. The args argument can either be
      * an array of arguments (like the onese passed on a command-line),
@@ -105,11 +113,13 @@ function (lang,   logger,   file,          parse,    optimize,   pragma,
 
     build._run = function (cmdConfig) {
         var buildFileContents = "",
+            pluginCollector = {},
             buildPaths, fileName, fileNames,
             prop, paths, i,
             baseConfig, config,
             modules, builtModule, srcPath, buildContext,
-            destPath;
+            destPath, moduleName, moduleMap, parentModuleMap, context,
+            resources, resource, pluginProcessed = {}, plugin;
 
         //Can now run the patches to require.js to allow it to be used for
         //build generation. Do it here instead of at the top of the module
@@ -294,9 +304,69 @@ function (lang,   logger,   file,          parse,    optimize,   pragma,
             //JS optimizations.
             fileNames = file.getFilteredFileList(config.dir, /\.js$/, true);
             for (i = 0; (fileName = fileNames[i]); i++) {
-                optimize.jsFile(fileName, fileName, config);
+                //Generate the module name from the config.dir root.
+                moduleName = fileName.replace(config.dir, '');
+                //Get rid of the extension
+                moduleName = moduleName.substring(0, moduleName.length - 3);
+                optimize.jsFile(fileName, fileName, config, moduleName, pluginCollector);
             }
 
+            //Normalize all the plugin resources.
+            context = require.s.contexts._;
+
+            for (moduleName in pluginCollector) {
+                if (pluginCollector.hasOwnProperty(moduleName)) {
+                    parentModuleMap = context.makeModuleMap(moduleName);
+                    resources = pluginCollector[moduleName];
+                    for (i = 0; (resource = resources[i]); i++) {
+                        moduleMap = context.makeModuleMap(resource, parentModuleMap);
+                        if (!context.plugins[moduleMap.prefix]) {
+                            //Set the value in context.plugins so it
+                            //will be evaluated as a full plugin.
+                            context.plugins[moduleMap.prefix] = true;
+
+                            //Do not bother if the plugin is not available.
+                            if (!file.exists(require.toUrl(moduleMap.prefix + '.js'))) {
+                                continue;
+                            }
+
+                            //Rely on the require in the build environment
+                            //to be synchronous
+                            context.require([moduleMap.prefix]);
+
+                            //Now that the plugin is loaded, redo the moduleMap
+                            //since the plugin will need to normalize part of the path.
+                            moduleMap = context.makeModuleMap(resource, parentModuleMap);
+                        }
+
+                        //Only bother with plugin resources that can be handled
+                        //processed by the plugin, via support of the writeFile
+                        //method.
+                        if (!pluginProcessed[moduleMap.fullName]) {
+                            //Only do the work if the plugin was really loaded.
+                            //Using an internal access because the file may
+                            //not really be loaded.
+                            plugin = context.defined[moduleMap.prefix];
+                            if (plugin && plugin.writeFile) {
+                                plugin.writeFile(
+                                    moduleMap.prefix,
+                                    moduleMap.name,
+                                    require,
+                                    writeFile,
+                                    context.config
+                                );
+                            }
+
+                            pluginProcessed[moduleMap.fullName] = true;
+                        }
+                    }
+
+                }
+            }
+
+            //console.log('PLUGIN COLLECTOR: ' + JSON.stringify(pluginCollector, null, "  "));
+
+
             //All module layers are done, write out the build.txt file.
             file.saveUtf8File(config.dir + "build.txt", buildFileContents);
         }

build/jslib/optimize.js

@@ -7,8 +7,10 @@
 /*jslint plusplus: false, nomen: false, regexp: false, strict: false */
 /*global define: false */
 
-define([ 'lang', 'logger', 'env!env/optimize', 'env!env/file', 'uglifyjs/index'],
-function (lang,   logger,   envOptimize,        file,           uglify) {
+define([ 'lang', 'logger', 'env!env/optimize', 'env!env/file', 'parse',
+         'uglifyjs/index'],
+function (lang,   logger,   envOptimize,        file,           parse,
+          uglify) {
 
     var optimize,
         cssImportRegExp = /\@import\s+(url\()?\s*([^);]+)\s*(\))?([\w, ]*)(;)?/g,
@@ -127,23 +129,45 @@ function (lang,   logger,   envOptimize,        file,           uglify) {
 
     optimize = {
         /**
-         * Optimizes a file that contains JavaScript content. It will inline
-         * text plugin files and run it through Google Closure Compiler
-         * minification, if the config options specify it.
+         * Optimizes a file that contains JavaScript content. Optionally collects
+         * plugin resources mentioned in a file, and then passes the content
+         * through an minifier if one is specified via config.optimize.
          *
          * @param {String} fileName the name of the file to optimize
          * @param {String} outFileName the name of the file to use for the
          * saved optimized content.
          * @param {Object} config the build config object.
+         * @param {String} [moduleName] the module name to use for the file.
+         * Used for plugin resource collection.
+         * @param {Array} [pluginCollector] storage for any plugin resources
+         * found.
          */
-        jsFile: function (fileName, outFileName, config) {
+        jsFile: function (fileName, outFileName, config, moduleName, pluginCollector) {
             var parts = (config.optimize + "").split('.'),
                 optimizerName = parts[0],
                 keepLines = parts[1] === 'keepLines',
-                fileContents, optFunc;
+                fileContents, optFunc, deps, i, dep;
 
             fileContents = file.readFile(fileName);
 
+            //If there is a plugin collector, scan the file for plugin resources.
+            if (config.optimizeAllPluginResources && pluginCollector) {
+                try {
+                    deps = parse.findDependencies(fileName, fileContents);
+                    if (deps.length) {
+                        for (i = 0; (dep = deps[i]); i++) {
+                            if (dep.indexOf('!') !== -1) {
+                                (pluginCollector[moduleName] ||
+                                 (pluginCollector[moduleName] = [])).push(dep);
+                            }
+                        }
+                    }
+                } catch (e) {
+                    logger.error('Parse error looking for plugin resources in ' +
+                                 fileName + ', skipping.');
+                }
+            }
+
             //Optimize the JS files if asked.
             if (optimizerName && optimizerName !== 'none') {
                 optFunc = envOptimize[optimizerName] || optimize.optimizers[optimizerName];

build/jslib/parse.js

@@ -37,31 +37,46 @@ define(['uglifyjs/index'], function (uglify) {
 
     /**
      * Validates a node as being an object literal (like for i18n bundles)
-     * or an array literal with just string members.
+     * or an array literal with just string members. If an array literal,
+     * only return array members that are full strings. So the caller of
+     * this function should use the return value as the new value for the
+     * node.
+     *
      * This function does not need to worry about comments, they are not
      * present in this AST.
+     *
+     * @param {Node} node an AST node.
+     *
+     * @returns {Node} an AST node to use for the valid dependencies.
+     * If null is returned, then it means the input node was not a valid
+     * dependency.
      */
     function validateDeps(node) {
-        var arrayArgs, i, dep;
+        var newDeps = ['array', []],
+            arrayArgs, i, dep;
+
+        if (!node) {
+            return null;
+        }
 
         if (isObjectLiteral(node) || node[0] === 'function') {
-            return true;
+            return node;
         }
 
         //Dependencies can be an object literal or an array.
         if (!isArrayLiteral(node)) {
-            return false;
+            return null;
         }
 
         arrayArgs = node[1];
 
         for (i = 0; i < arrayArgs.length; i++) {
             dep = arrayArgs[i];
-            if (dep[0] !== 'string') {
-                return false;
+            if (dep[0] === 'string') {
+                newDeps[1].push(dep);
             }
         }
-        return true;
+        return newDeps[1].length ? newDeps : null;
     }
 
     /**
@@ -77,7 +92,12 @@ define(['uglifyjs/index'], function (uglify) {
         var matches = [], result = null,
             astRoot = parser.parse(fileContents);
 
-        parse.recurse(astRoot, matches);
+        parse.recurse(astRoot, function () {
+            var parsed = parse.callToString.apply(parse, arguments);
+            if (parsed) {
+                matches.push(parsed);
+            }
+        });
 
         if (matches.length) {
             result = matches.join("\n");
@@ -94,19 +114,16 @@ define(['uglifyjs/index'], function (uglify) {
     /**
      * Handles parsing a file recursively for require calls.
      * @param {Array} parentNode the AST node to start with.
-     * @param {Array} matches where to store the string matches
+     * @param {Function} onMatch function to call on a parse match.
      */
-    parse.recurse = function (parentNode, matches) {
-        var i, node, parsed;
+    parse.recurse = function (parentNode, onMatch) {
+        var i, node;
         if (isArray(parentNode)) {
             for (i = 0; i < parentNode.length; i++) {
                 node = parentNode[i];
                 if (isArray(node)) {
-                    parsed = this.parseNode(node);
-                    if (parsed) {
-                        matches.push(parsed);
-                    }
-                    this.recurse(node, matches);
+                    this.parseNode(node, onMatch);
+                    this.recurse(node, onMatch);
                 }
             }
         }
@@ -192,6 +209,42 @@ define(['uglifyjs/index'], function (uglify) {
         return null;
     };
 
+    /**
+     * Finds all dependencies specified in dependency arrays and inside
+     * simplified commonjs wrappers.
+     * @param {String} fileName
+     * @param {String} fileContents
+     *
+     * @returns {Array} an array of dependency strings. The dependencies
+     * have not been normalized, they may be relative IDs.
+     */
+    parse.findDependencies = function (fileName, fileContents) {
+        //This is a litle bit inefficient, it ends up with two uglifyjs parser
+        //calls. Can revisit later, but trying to build out larger functional
+        //pieces first.
+        var dependencies = parse.getAnonDeps(fileName, fileContents),
+            astRoot = parser.parse(fileContents),
+            i, dep;
+
+        parse.recurse(astRoot, function (callName, config, name, deps) {
+            //Normalize the input args.
+            if (name && isArrayLiteral(name)) {
+                deps = name;
+                name = null;
+            }
+
+            if (!(deps = validateDeps(deps)) || !isArrayLiteral(deps)) {
+                return;
+            }
+
+            for (i = 0; (dep = deps[1][i]); i++) {
+                dependencies.push(dep[1]);
+            }
+        });
+
+        return dependencies;
+    };
+
     parse.findRequireDepNames = function (node, deps) {
         var moduleName, i, n, call, args;
 
@@ -291,7 +344,7 @@ define(['uglifyjs/index'], function (uglify) {
             name = null;
         }
 
-        if (deps && !validateDeps(deps)) {
+        if (!(deps = validateDeps(deps))) {
             return null;
         }
 
@@ -311,11 +364,14 @@ define(['uglifyjs/index'], function (uglify) {
     /**
      * Determines if a specific node is a valid require or define/require.def call.
      * @param {Array} node
+     * @param {Function} onMatch a function to call when a match is found.
+     * It is passed the match name, and the config, name, deps possible args.
+     * The config, name and deps args are not normalized.
      *
      * @returns {String} a JS source string with the valid require/define call.
      * Otherwise null.
      */
-    parse.parseNode = function (node) {
+    parse.parseNode = function (node, onMatch) {
         var call, name, config, deps, args;
 
         if (!isArray(node)) {
@@ -337,11 +393,11 @@ define(['uglifyjs/index'], function (uglify) {
                         config = null;
                     }
 
-                    if (!deps || !validateDeps(deps)) {
+                    if (!(deps = validateDeps(deps))) {
                         return null;
                     }
 
-                    return this.callToString("require", null, null, deps);
+                    return onMatch("require", null, null, deps);
 
                 } else if ((call[0] === 'name' && call[1] === 'define') ||
                            (call[0] === 'dot' && call[1][1] === 'require' &&
@@ -364,7 +420,7 @@ define(['uglifyjs/index'], function (uglify) {
                           name[0] === 'function' || isObjectLiteral(name))) &&
                         (!deps || isArrayLiteral(deps) ||
                          deps[0] === 'function' || isObjectLiteral(deps))) {
-                        return this.callToString("define", null, name, deps);
+                        return onMatch("define", null, name, deps);
                     }
                 }
             }

build/tests/parse.js

@@ -13,7 +13,7 @@ define(['parse'], function (parse) {
 
                 t.is('require(["one","two"]);', parse("good1", good1));
                 t.is('require(["one","two"]);', parse("good2", good2));
-                t.is(null, parse("bad1", bad1));
+                t.is('require(["me"]);', parse("bad1", bad1));
                 t.is(null, parse("bad2", bad2));
             }
         ]
@@ -38,7 +38,7 @@ define(['parse'], function (parse) {
                 t.is('define("one",function(){});', parse("good2", good2));
                 t.is('define("one",["two"]);', parse("good3", good3));
                 t.is('define("one",function(){});', parse("good4", good4));
-                t.is(null, parse("bad1", bad1));
+                t.is('define("one",["me"]);', parse("bad1", bad1));
                 t.is(null, parse("bad2", bad2));
                 t.is(['require', 'foo'], parse.getAnonDeps("goodAnon1", goodAnon1));
                 t.is(['require', 'exports', 'module', 'bar'], parse.getAnonDeps("goodAnon2", goodAnon2));
@@ -70,8 +70,8 @@ define(['parse'], function (parse) {
                 t.is('define("one",function(){});', parse("good2", good2));
                 t.is('define("one",["two"]);', parse("good3", good3));
                 t.is('define("one",function(){});', parse("good4", good4));
-                t.is('define("foo",[]);', parse("nested1", nested1));
-                t.is(null, parse("bad1", bad1));
+                t.is(null, parse("nested1", nested1));
+                t.is('define("one",["me"]);', parse("bad1", bad1));
                 t.is(null, parse("bad2", bad2));
                 t.is(null, parse("bad3", bad3));
                 t.is(null, parse("bad4", bad4));

build/tests/text.build.js

@@ -1,11 +1,12 @@
 //A simple build file using the tests directory for requirejs
 {
-    baseUrl: "../../tests/text",
+    baseUrl: "../../../requirejs/tests/text",
     paths: {
         text: "../../text"
     },
     dir: "builds/text",
     optimize: "none",
+    optimizeAllPluginResources: true,
 
     modules: [
         {

require.js

@@ -112,6 +112,22 @@ var require, define;
         }
     }
 
+    /**
+     * jQuery 1.4.3-1.5.x use a readyWait/ready() pairing to hold DOM
+     * ready callbacks, but jQuery 1.6 supports a holdReady() API instead.
+     * At some point remove the readyWait/ready() support and just stick
+     * with using holdReady.
+     */
+    function jQueryHoldReady($, shouldHold) {
+        if ($.holdReady) {
+            $.holdReady(shouldHold);
+        } else if (shouldHold) {
+            $.readyWait += 1;
+        } else {
+            $.ready(true);
+        }
+    }
+
     //Check for an existing version of require. If so, then exit out. Only allow
     //one version of require to be active in a page. However, allow for a require
     //config object, just exit quickly if require is an actual function.
@@ -690,9 +706,9 @@ var require, define;
         }
 
         /**
-         * As of jQuery 1.4.3, it supports a readyWait property that will hold off
+         * jQuery 1.4.3+ supports ways to hold off calling
          * calling jQuery ready callbacks until all scripts are loaded. Be sure
-         * to track it if readyWait is available. Also, since jQuery 1.4.3 does
+         * to track it if the capability exists.. Also, since jQuery 1.4.3 does
          * not register as a module, need to do some global inference checking.
          * Even if it does register as a module, not guaranteed to be the precise
          * name of the global. If a jQuery is tracked for this context, then go
@@ -709,7 +725,7 @@ var require, define;
                         return;
                     }
 
-                    if ("readyWait" in $) {
+                    if ("holdReady" in $ || "readyWait" in $) {
                         context.jQuery = $;
 
                         //Manually create a "jquery" module entry if not one already
@@ -720,9 +736,9 @@ var require, define;
                             return jQuery;
                         }]);
 
-                        //Increment jQuery readyWait if ncecessary.
+                        //Ask jQuery to hold DOM ready callbacks.
                         if (context.scriptCount) {
-                            $.readyWait += 1;
+                            jQueryHoldReady($, true);
                             context.jQueryIncremented = true;
                         }
                     }
@@ -1160,7 +1176,7 @@ var require, define;
             },
 
             require: function (deps, callback, relModuleMap) {
-                var moduleName, ret, moduleMap;
+                var moduleName, fullName, moduleMap;
                 if (typeof deps === "string") {
                     //Synchronous access to one module. If require.get is
                     //available (as in the Node adapter), prefer that.
@@ -1177,15 +1193,15 @@ var require, define;
 
                     //Normalize module name, if it contains . or ..
                     moduleMap = makeModuleMap(moduleName, relModuleMap);
+                    fullName = moduleMap.fullName;
 
-                    ret = defined[moduleMap.fullName];
-                    if (ret === undefined) {
+                    if (!(fullName in defined)) {
                         return req.onError(makeError("notloaded", "Module name '" +
                                     moduleMap.fullName +
                                     "' has not been loaded yet for context: " +
                                     contextName));
                     }
-                    return ret;
+                    return defined[fullName];
                 }
 
                 main(null, deps, callback, relModuleMap);
@@ -1428,6 +1444,14 @@ var require, define;
         return context.require(deps, callback);
     };
 
+    /**
+     * Global require.toUrl(), to match global require, mostly useful
+     * for debugging/work in the global space.
+     */
+    req.toUrl = function (moduleNamePlusExt) {
+        return contexts[defContextName].toUrl(moduleNamePlusExt);
+    };
+
     req.version = version;
     req.isArray = isArray;
     req.isFunction = isFunction;
@@ -1485,11 +1509,11 @@ var require, define;
         context.scriptCount += 1;
         req.attach(url, contextName, moduleName);
 
-        //If tracking a jQuery, then make sure its readyWait
-        //is incremented to prevent its ready callbacks from
+        //If tracking a jQuery, then make sure its ready callbacks
+        //are put on hold to prevent its ready callbacks from
         //triggering too soon.
         if (context.jQuery && !context.jQueryIncremented) {
-            context.jQuery.readyWait += 1;
+            jQueryHoldReady(context.jQuery, true);
             context.jQueryIncremented = true;
         }
     };
@@ -1820,14 +1844,13 @@ var require, define;
                 }
             }
 
-            //If jQuery with readyWait is being tracked, updated its
-            //readyWait count.
+            //If jQuery with DOM ready delayed, release it now.
             contexts = s.contexts;
             for (prop in contexts) {
                 if (!(prop in empty)) {
                     context = contexts[prop];
                     if (context.jQueryIncremented) {
-                        context.jQuery.ready(true);
+                        jQueryHoldReady(context.jQuery, false);
                         context.jQueryIncremented = false;
                     }
                 }

tasks.txt

@@ -1 +1,2 @@
+* Need more docs on optimizeAllPluginResources?
 * allow for npm install