Cleaned up names and added instructions

Author: hatton

.gitignore

@@ -1,4 +1,5 @@
-# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+output/
+samples/
 
 # Logs
 

README.md

@@ -0,0 +1,33 @@
+In Bloom 5.7, we introduce a modernized, parameterized page layout CSS system, known as "Appearance". This new system conflicts with the arbitrary custom css that was used in prior Bloom versions, mostly in the area of margins.
+
+As of this writing, we are only looking at `customBookStyles.css`, but `customCollectionsStyles.css` have the same problem.
+
+When Bloom encounters a pre-5.7 book with a `customBookStyles.css` that would conflict with the new system, it goes into a kind of "safe mode" that keeps things working by using a legacy `basePage.css`. Better, though, is to migrate the old css to the new system. Conceivably, a reliable program could be written to do this automatically. However at the moment what we do is to write the migrations using the utilities here, and then have a dev evaluate individual migrations and then copy them over to the shipping Bloom. So when Bloom encounters one of these books, we may already have a tested migration for it.
+
+# How to use this system
+
+⚠️ Be careful what you commit to an open source repo. We do not want to expose emails or other private data.
+
+1.  As of this writing, bun only works on linux. If you are on windows, just install Windows Subsystem for Linux (not as big of a deal as it sounds), then run in a WSL terminal in VSCODE.
+
+1.  Get [bun](https://bun.sh/) installed
+
+1.  Install dependencies: `bun install`
+
+1.  Download all the `customBookStyles.css` from blorg
+
+    `./some-path/BloomBulkDownloader.exe --include "*/*/custom*Styles.css" --syncfolder ./output/downloads --bucket production customBookStyles-files`
+
+1.  Process those, grouping them into bins of duplicate stylesheets
+
+    `bun run group-stylesheets.ts 13`
+
+    Here the number is optional and it limits how many stylesheets will be processed.
+
+1.  Process those groups, discarding ones that don't need migration
+
+    `bun run filter-stylesheets.ts`
+
+1.  Create draft migration files for each one
+
+    `bun run create-migrations.ts`

custom-css-analysis-and-migration/README.md

@@ -0,0 +1,9 @@
+// This reads in a json of the unique custom sets of rules that need migration.
+// For each one, it outputs a folder that serves as a draft "migration" for Bloom.
+// See README.md for use.
+
+import fs from "fs";
+
+const data = fs.readFileSync("./output/filter-output.json", "utf8");
+
+// TODO: see https://issues.bloomlibrary.org/youtrack/issue/BL-12857

custom-css-analysis-and-migration/create-migrations.ts

@@ -1,11 +1,14 @@
+// This reads in a json of all unique custom sets of rules, drops the ones that don't need to be migrated,
+// and then outputs the results as json file. See README.md for use.
+
 import fs from "fs";
 
 interface Record {
   content: string;
   paths: string[];
 }
 
-const data = fs.readFileSync("have-custom.json", "utf8");
+const data = fs.readFileSync("./output/group-output.json", "utf8");
 const records: Record[] = JSON.parse(data);
 
 const count = records.reduce((acc, record) => acc + record.paths.length, 0);
@@ -24,21 +27,37 @@ const recordsWithUniqueifiedPaths = filteredRecords.map((record) => {
   const uniquePaths = uniqueFilenames.map((filename) => {
     return paths.find((path) => path.endsWith(filename));
   });
+  const instanceId = paths[0].split("/")[2];
   return {
     book_count: paths.length,
     unique_named_books: uniquePaths.length,
+    first_book: `https://bloomlibrary.org/:search:bookInstanceId%3A${instanceId}`,
     css: record.content,
     paths,
     uniqueified_paths: uniquePaths,
   };
 });
 
 const sortedRecords = recordsWithUniqueifiedPaths.sort((a, b) => {
-  return a.uniqueified_paths.length - b.uniqueified_paths.length;
+  return b.uniqueified_paths.length - a.uniqueified_paths.length;
+});
+
+// insert a metadata record into the first position
+(sortedRecords as any).unshift({
+  "total books with custom css rules": count,
+  "total books with problematic rules": filteredRecords.reduce(
+    (acc, record) => acc + record.paths.length,
+    0
+  ),
+  "total unique css files": records.length,
+  "unique CSS files with problematic rules": filteredRecords.length,
+  "counts of unique books for each unique css file": sortedRecords
+    .map((record) => record.uniqueified_paths.length)
+    .join(" "),
 });
 
 fs.writeFileSync(
-  "problematic-rules.json",
+  "./output/filter-output.json",
   JSON.stringify(sortedRecords, null, 2)
 );
 
@@ -52,9 +71,9 @@ console.write(
   )} books, `
 );
 
-console.write(
-  `${recordsWithUniqueifiedPaths.reduce(
-    (acc, record) => acc + record.uniqueified_paths.length,
-    0
-  )} of which have unique names (should remove most rebrands).\r\n`
-);
+// console.write(
+//   `${recordsWithUniqueifiedPaths.reduce(
+//     (acc, record) => acc + record.uniqueified_paths.length,
+//     0
+//   )} of which have unique names (should remove most rebrands).\r\n`
+// );

custom-css-analysis-and-migration/filter-to-problematic-css-rulesets.ts renamed to custom-css-analysis-and-migration/filter-stylesheets.ts

@@ -1,3 +1,7 @@
+// This looks at all the stylesheets, groups them into unique sets of rules,
+// and then outputs the results as json file. See README.md for use.
+// It takes a single optional argument, which is the number of books to process.
+
 import * as fs from "fs";
 import * as path from "path";
 
@@ -19,7 +23,7 @@ function readFilesRecursively(
   const files = fs.readdirSync(dir);
   for (const file of files) {
     const filePath = path.join(dir, file);
-    if (count > 100000) return;
+    if (Bun.argv.length > 2 && count >= Number.parseInt(Bun.argv[2])) return;
     if (fs.statSync(filePath).isDirectory()) {
       readFilesRecursively(filePath, fileMap);
     } else {
@@ -31,30 +35,25 @@ function readFilesRecursively(
       if (content === "") continue;
 
       const fileData: FileData = fileMap.get(content) || { content, paths: [] };
-      fileData.paths.push(
-        dir.replace("/mnt/c/dev/BloomBulkDownloader/sync", "")
-      );
+      fileData.paths.push(dir.replace("./output/downloads", ""));
       fileMap.set(content, fileData);
       console.log(++count + " " + dir);
     }
   }
 }
 
-function writeUniqueFiles(dir: string, fileMap: Map<string, FileData>): void {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir);
-  }
-  const indexFilePath = path.join(dir, "index.json");
-  const indexFileContent = JSON.stringify(
-    Array.from(fileMap.values()),
-    null,
-    2
-  );
-  fs.writeFileSync(indexFilePath, indexFileContent);
+const sourceDir = "./output/downloads";
+const fileMap = new Map<string, FileData>();
+
+if (!fs.existsSync("./output")) {
+  fs.mkdirSync("./output");
 }
 
-const sourceDir = "/mnt/c/dev/BloomBulkDownloader/sync";
-const targetDir = "/mnt/c/dev/BloomBulkDownloader/unique";
-const fileMap = new Map<string, FileData>();
+if (fs.existsSync("./output/group-output.json")) {
+  fs.rmSync("./output/group-output.json");
+}
 readFilesRecursively(sourceDir, fileMap);
-writeUniqueFiles(targetDir, fileMap);
+fs.writeFileSync(
+  "./output/group-output.json",
+  JSON.stringify(Array.from(fileMap.values()), null, 2)
+);

custom-css-analysis-and-migration/analyze-custom-css-files.ts renamed to custom-css-analysis-and-migration/group-stylesheets.ts

@@ -1,3 +1,10 @@
+/**
+ *
+ * I'm not clear what state this is in, it was an experiment I was working on.
+ * It probably should become a function that is called by create-migrations.ts
+ *
+ * **/
+
 import fs from "fs";
 import { parse, stringify, Rule, Stylesheet, Declaration } from "css";
 
@@ -52,6 +59,21 @@ const sizes = [
   { name: ".Device16x9Landscape", height: 100, width: 177.77777778 },
   { name: ".PictureStoryLandscape", height: 100, width: 177.77777778 },
 ];
+interface PropertyConversions {
+  [key: string]: string;
+}
+const propertyConversions: PropertyConversions = {
+  left: "--page-margin-left",
+  top: "--page-margin-top",
+  height: "--page-margin-bottom",
+  width: "--page-margin-right",
+};
+const coverPropertyConversions: PropertyConversions = {
+  left: "--page-margin-left",
+  top: "--cover-margin-top",
+  height: "--cover-margin-bottom",
+  width: "--page-margin-right",
+};
 
 const cssFileContent: string = fs.readFileSync("./zebra.css", "utf8");
 const cssObject: Stylesheet = parse(cssFileContent);
@@ -94,9 +116,7 @@ if (cssObject.stylesheet && cssObject.stylesheet.rules) {
       rule.declarations?.forEach((declaration: Declaration) => {
         if (declaration.type === "declaration") {
           const key = (declaration as Declaration).property;
-          interface PropertyConversions {
-            [key: string]: string;
-          }
+
           if (size) {
             if (key === "width") {
               declaration.value =
@@ -113,22 +133,85 @@ if (cssObject.stylesheet && cssObject.stylesheet.rules) {
                 "mm";
             }
           }
-          const propertyConversions: PropertyConversions = {
-            left: "--page-margin-left",
-            top: "--page-margin-top",
-            height: "--page-margin-bottom",
-            width: "--page-margin-right",
-          };
-
-          if (declaration.property! in propertyConversions) {
-            declaration.property = propertyConversions[declaration.property!];
+
+          const isCover = rule.selectors!.some((sel) => sel.includes("Cover"));
+          const map = isCover ? coverPropertyConversions : propertyConversions;
+          if (declaration.property! in map) {
+            declaration.property = map[declaration.property!];
             declaration.value = declaration.value?.replace("!important", "");
           }
         }
       });
+
+      // danger, this would probably break if there is anything but classes in the selector
+      sortClassesInSelector(rule);
+
+      // TODO: this doesn't yet move top and bottom margins with .outsideFrontCover and .outsideBackCover to --cover-margin-top and --cover-margin-bottom
+
+      sortDeclarations(rule);
     }
   });
+
+  // danger, normally sorting rules is not a good idea!
+  cssObject.stylesheet.rules = sortRules(cssObject.stylesheet.rules);
 }
 
 const modifiedCss: string = stringify(cssObject);
 console.log(modifiedCss);
+
+function sortDeclarations(rule: Rule) {
+  // Define the type of propertyConversions
+  type PropertyConversions = {
+    [key: string]: number;
+  };
+
+  // Define the property conversions object
+  const orderedProperties: PropertyConversions = {
+    "--page-margin-top": 0,
+    "--page-margin-bottom": 1,
+    "--page-margin-left": 2,
+    "--page-margin-right": 3,
+  };
+
+  // sort the declarations according to the order in propertyConversions
+  rule.declarations?.sort((a: Declaration, b: Declaration) => {
+    const aProp = orderedProperties[a.property!];
+    const bProp = orderedProperties[b.property!];
+    if (aProp === undefined && bProp === undefined) {
+      return 0;
+    } else if (aProp === undefined) {
+      return 1;
+    } else if (bProp === undefined) {
+      return -1;
+    } else {
+      return aProp - bProp;
+    }
+  });
+}
+function sortClassesInSelector(rule: any): void {
+  // sort the classes in the first selector
+  const classes = rule.selectors[0].trim().split(".").filter(Boolean).sort();
+  const sortedSelector = "." + classes.join(".");
+  rule.selectors[0] = sortedSelector;
+}
+
+function sortRules(rules: Rule[]): Rule[] {
+  return rules.sort((a: Rule, b: Rule) => {
+    if (a.type !== "rule" || b.type !== "rule") return 0;
+
+    // sort rules by first selector
+    const aSelector = a.selectors ? a.selectors[0] : undefined;
+    const bSelector = b.selectors ? b.selectors[0] : undefined;
+    if (
+      aSelector === bSelector ||
+      aSelector === undefined ||
+      bSelector === undefined
+    ) {
+      return 0;
+    } else if (aSelector > bSelector) {
+      return 1;
+    } else {
+      return -1;
+    }
+  });
+}