find example scripts properly again

Author: jefdaj

shortcut

@@ -44,7 +44,7 @@
 from glob                import glob
 from jinja2              import ChoiceLoader, FileSystemLoader
 from misaka              import Markdown, HtmlRenderer
-from os                  import setsid, getpgid, killpg, makedirs, symlink, readlink
+from os                  import setsid, getpgid, killpg, makedirs, symlink, readlink, walk
 from os.path             import exists, join, relpath, realpath, dirname, basename, splitext
 from pexpect             import spawn, EOF
 from psutil              import cpu_percent, virtual_memory
@@ -172,11 +172,18 @@ def blockcode(self, text, lang):
 #     #print 'names:', names
 #     return names
 
+def find_cuts(dirpath):
+    for root, dirs, files in walk(dirpath, followlinks=True):
+        # see https://stackoverflow.com/a/19859907
+        dirs[:] = [d for d in dirs if not d.startswith('.') and not basename(d) == 'tmpdir']
+        for file in files:
+            if file.endswith('.cut'):
+                yield(join(root, file))
+
 def load_codeblocks():
     # used to render the code examples
     codeblocks = {}
-    # for path in glob(join(CONFIG['examples_dir'], '*.cut')) + glob(join(CONFIG['users_dir'], '*/*.cut')):
-    for path in glob(join(CONFIG['users_dir'], '*/*.cut')) + glob(join(CONFIG['users_dir'], '*/*/*.cut')):
+    for path in find_cuts(CONFIG['users_dir']):
         with open(path, 'r') as f:
             txt = '```\n%s\n```\n' % f.read()
             # name = basename(path)
@@ -190,7 +197,7 @@ def load_codeblocks():
             , 'content': MARKDOWN(txt)
             }
     # print codeblocks
-    # print codeblocks.keys()
+    LOGGER.info('loaded codeblocks: %s' % codeblocks.keys())
     return codeblocks
 
 def load_tutorial_sections():
@@ -264,7 +271,7 @@ def create_user(username, password):
 def list_user_scripts(username):
     # load_code_blocks() # super hacky but reloads code blocks along with templates
     upath = join(CONFIG['users_dir'], username)
-    paths = [relpath(p, upath) for p in glob(join(upath, '*.cut')) + glob(join(upath, '*/*.cut'))]
+    paths = [relpath(p, upath) for p in find_cuts(upath)]
     # paths.sort()
     return paths
 

shortcut-demo.py

@@ -48,14 +48,14 @@ or make an account to save your own privately.
 There are two types of interactive code blocks.
 Complete cut scripts with `Load` buttons like this:
 
-{{ macros.load_cut(user, 'examples/scripts/load03.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/load03.cut') }}
 
 ... and examples of other commands you would type in the terminal.
 The `Run` button just types them for you.
 They can include loading scripts, but also anything else you might do live:
 redefine variables, look at depdencies, etc.
 
-{{ macros.run_example([':load examples/scripts/load03.cut', 'sample 10 genes_of_interest', ':type']) }}
+{{ macros.run_example([':load examples/cut-scripts/load03.cut', 'sample 10 genes_of_interest', ':type']) }}
 
 
 # Using the command line program

templates/basics.md

@@ -1,6 +1,6 @@
 {% import "macros.jinja" as macros with context %}
 
-This page is a complete listing of available functions in ShortCut v0.9.1. The search box only filters by module. So for example if you search for "mmseqs", you'll get the MMSeqs module but also BlastHits and ListLike, because they can use MMSeqs results.
+This page is a complete listing of available functions in ShortCut v0.9.2. The search box only filters by module. So for example if you search for "mmseqs", you'll get the MMSeqs module but also BlastHits and ListLike, because they can use MMSeqs results.
 
 Each module has (or will have soon!) an example script making use of all its functions.
 If you don't find what you're looking for, leave Jeff a comment about it.
@@ -27,7 +27,7 @@ Functions:
 | `replace_each` | `<outputvar>`, `<inputvar>`, `<inputvars>` | `<output>.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/replace.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/replace.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Repeat module</h3>
@@ -42,7 +42,7 @@ Functions:
 | `repeat` | `<outputvar>`, `<inputvar>`, `num` | `<output>.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/repeat.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/repeat.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Math module</h3>
@@ -60,7 +60,7 @@ Functions:
 | `/` | `num`, `num` | `num` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/math.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/math.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Load module</h3>
@@ -76,7 +76,7 @@ Functions:
 | `glob_files` | `str` | `str.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/load.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/load.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Sets module</h3>
@@ -97,7 +97,7 @@ Functions:
 | `diff` | `X.list.list` | `X.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/sets.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/sets.cut') }}
 </div>
 <div class="moduleblock">
 <h3>SeqIO module</h3>
@@ -146,7 +146,7 @@ Functions:
 | `load_gbk_glob` | `str` | `gbk.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/seqio.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/seqio.cut') }}
 </div>
 <div class="moduleblock">
 <h3>BiomartR module</h3>
@@ -170,7 +170,7 @@ Functions:
 | `get_proteomes` | `str.list` | `faa.gz.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/biomartr.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/biomartr.cut') }}
 </div>
 <div class="moduleblock">
 <h3>BlastDB module</h3>
@@ -204,7 +204,7 @@ Functions:
 | `singletons` | `X.list` | `X.list.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/blastdb.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/blastdb.cut') }}
 </div>
 <div class="moduleblock">
 <h3>BLAST+ module</h3>
@@ -251,7 +251,7 @@ Functions:
 | `concat_bht_each` | `bht.list.list` | `bht.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/blast.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/blast.cut') }}
 </div>
 <div class="moduleblock">
 <h3>BlastHits module</h3>
@@ -284,7 +284,7 @@ Functions:
 | `best_hits_each` | `hittable.list` | `bht.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/blasthits.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/blasthits.cut') }}
 </div>
 <div class="moduleblock">
 <h3>ListLike module</h3>
@@ -308,7 +308,7 @@ Functions:
 | `length_each` | `listlike.list` | `num.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/listlike.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/listlike.cut') }}
 </div>
 <div class="moduleblock">
 <h3>PsiBLAST module</h3>
@@ -389,7 +389,7 @@ Functions:
 | `psiblast_train_pssms_db` | `num`, `faa.list`, `pdb` | `pssm.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/psiblast.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/psiblast.cut') }}
 </div>
 <div class="moduleblock">
 <h3>CRB-BLAST module</h3>
@@ -413,7 +413,7 @@ Functions:
 | `crb_blast_each` | `fna`, `fa.list` | `crb.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/crbblast.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/crbblast.cut') }}
 </div>
 <div class="moduleblock">
 <h3>HMMER module</h3>
@@ -441,7 +441,7 @@ Functions:
 | `extract_hmm_targets_each` | `hht.list` | `str.list.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/hmmer.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/hmmer.cut') }}
 </div>
 <div class="moduleblock">
 <h3>BlastRBH module</h3>
@@ -481,7 +481,7 @@ Functions:
 | `tblastx_rbh_each` | `num`, `fna`, `fna.list` | `bht.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/blastrbh.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/blastrbh.cut') }}
 </div>
 <div class="moduleblock">
 <h3>MUSCLE module</h3>
@@ -503,7 +503,7 @@ Functions:
 | `muscle_each` | `faa.list` | `aln.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/muscle.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/muscle.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Sample module</h3>
@@ -522,7 +522,7 @@ Functions:
 | `sample` | `num`, `X.list` | `X.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/sample.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/sample.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Permute module</h3>
@@ -537,7 +537,7 @@ Functions:
 | `leave_each_out` | `X.list` | `X.list.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/permute.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/permute.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Summarize module</h3>
@@ -547,7 +547,7 @@ Collapse a list of results into a single summary.
 
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/summarize.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/summarize.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Scores module</h3>
@@ -564,7 +564,7 @@ Functions:
 | `extract_scored` | `X.scores` | `X.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/scores.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/scores.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Plots module</h3>
@@ -587,7 +587,7 @@ Functions:
 | `venndiagram` | `X.list.list` | `png` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/plots.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/plots.cut') }}
 </div>
 <div class="moduleblock">
 <h3>OrthoFinder module</h3>
@@ -608,7 +608,7 @@ Functions:
 | `orthofinder` | `faa.list` | `ofr` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/orthofinder.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/orthofinder.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Diamond module</h3>
@@ -674,7 +674,7 @@ Functions:
 | `diamond_blastx_db_more_sensitive_each` | `num`, `fna`, `dmnd.list` | `bht.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/diamond.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/diamond.cut') }}
 </div>
 <div class="moduleblock">
 <h3>MMSeqs module</h3>
@@ -700,7 +700,7 @@ Functions:
 | `mmseqs_search` | `num`, `fa`, `fa` | `bht` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/mmseqs.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/mmseqs.cut') }}
 </div>
 <div class="moduleblock">
 <h3>SonicParanoid module</h3>
@@ -722,7 +722,7 @@ Functions:
 | `sonicparanoid` | `faa.list` | `spr` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/sonicparanoid.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/sonicparanoid.cut') }}
 </div>
 <div class="moduleblock">
 <h3>OrthoGroups module</h3>
@@ -755,7 +755,7 @@ Functions:
 | `ortholog_in_max_str` | `num`, `str.list.list`, `str.list.list` | `str.list.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/orthogroups.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/orthogroups.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Busco module</h3>
@@ -789,7 +789,7 @@ Functions:
 | `concat_bst` | `bst.list` | `bst` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/busco.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/busco.cut') }}
 </div>
 <div class="moduleblock">
 <h3>Range module</h3>
@@ -813,7 +813,7 @@ Functions:
 | `range_multiply` | `num`, `num`, `num` | `num.list` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/range.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/range.cut') }}
 </div>
 <div class="moduleblock">
 <h3>SetsTable module</h3>
@@ -833,7 +833,7 @@ Functions:
 | `sets_table` | `lit.list.list` | `tsv` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/setstable.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/setstable.cut') }}
 </div>
 <div class="moduleblock">
 <h3>All-Vs-All module</h3>
@@ -848,7 +848,7 @@ Types:
 
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/allvsall.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/allvsall.cut') }}
 </div>
 <div class="moduleblock">
 <h3>GreenCut module</h3>
@@ -868,5 +868,5 @@ Functions:
 | `greencut2_families` | `bht`, `bht` | `gcr` |
 
 <br/>
-{{ macros.load_cut(user, 'examples/scripts/greencut.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/greencut.cut') }}
 </div>

templates/functions.md

@@ -3,6 +3,6 @@
 Now you know everything you need to make an actual, useful cut.
 Like, the kind you could publish!
 
-{{ macros.load_cut(user, 'examples/scripts/green.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/green.cut') }}
 
-{{ macros.load_cut(user, 'examples/scripts/green-ids.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/green-ids.cut') }}

templates/greencut.md

@@ -1,6 +1,6 @@
 {% import "macros.jinja" as macros with context %}
 
-Here are a bunch of example scripts, including everything from the rest of the site.
+Here are a bunch of example scripts, including everything from the rest of the (public) site.
 
 If you don't find an example similar to what you're trying to do, request that it be added (bottom right).
 Or if you figure it out yourself, consider adding your script to the collection!

templates/scripts.md

@@ -3,7 +3,7 @@
 Let's start at the beginning.
 This is probably the simplest script you could write:
 
-{{ macros.load_cut(user, 'examples/scripts/variables01.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/variables01.cut') }}
 
 <!-- TODO include example scripts in the main repo? either that or rewrite this: -->
 <!-- idea: :help <varname> should expand to help on its type -->
@@ -15,7 +15,7 @@ You'll spend most of your time editing it in the interpreter,
 defining and evaluating individual variables.
 Here is a script with several of them.
 
-{{ macros.load_cut(user, 'examples/scripts/variables02.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/variables02.cut') }}
 
 ShortCut keeps track of dependencies between variables, like this:
 

templates/tutorial_01_variables.md

@@ -6,7 +6,7 @@ operations.
 
 A few examples:
 
-{{ macros.load_cut(user, 'examples/scripts/math01.cut') }}
+{{ macros.load_cut(user, 'examples/cut-scripts/math01.cut') }}
 
 You can enter numbers in decimal or scientific notation. There's only one type
 of number instead of the several different ones like doubles and floats you