Skip to content

Commit e285faf

Browse files
mbuechsemaxwolfs
andauthored
Automate creation of standards overview pages (#116)
* Automate creation of standards overview pages resolves #97 Signed-off-by: Matthias Büchse <[email protected]> Signed-off-by: Max Wolfs <[email protected]> Co-authored-by: Max Wolfs <[email protected]>
1 parent c27f229 commit e285faf

29 files changed

+336
-592
lines changed

.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
/docs/04-operating-scs/components
1515
/docs/06-releases
1616
/standards/*.md
17+
/standards/*/*.md
18+
/standards/scs-*.yaml
1719

1820
# Dependencies
1921
node_modules
@@ -24,6 +26,8 @@ node_modules
2426
# Generated files
2527
.docusaurus
2628
.cache-loader
29+
sidebarsStandardsItems.js
30+
sidebarsCertificationItems.js
2731

2832
# Misc
2933
.DS_Store

docs.package.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@
2525
},
2626
{
2727
"repo": "SovereignCloudStack/standards",
28-
"source": "Standards/*.md",
28+
"source": [
29+
"Standards/*.md",
30+
"Tests/scs-*.yaml"
31+
],
2932
"target": "standards",
3033
"label": ""
3134
},

getDocs.js

+9-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,15 @@ repos.forEach((repo) => {
3131
fs.mkdirSync(subDirPath, { recursive: true })
3232

3333
// Copy docs content from A to B
34-
const copyDocsCommand = `cp -r ${repoDir}/${repo.source} ${subDirPath}`
35-
execSync(copyDocsCommand)
34+
// allow multiple sources here so the same repo need not be checked out multiple times
35+
// however, it would be better if this script automatically grouped all entries by repo and then only
36+
// checked out each repo only once; I leave this as a TODO because I don't fully grasp the meaning of
37+
// label, for instance, and the label is used for the temporary repo directory
38+
let sources = Array.isArray(repo.source) ? repo.source : [repo.source]
39+
sources.forEach((source) => {
40+
const copyDocsCommand = `cp -r ${repoDir}/${source} ${subDirPath}`
41+
execSync(copyDocsCommand)
42+
})
3643

3744
// Remove the cloned repository
3845
const removeRepoCommand = 'rm -rf repo_to_be_edited'

package-lock.json

+13-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"write-translations": "docusaurus write-translations",
3030
"write-heading-ids": "docusaurus write-heading-ids",
3131
"typecheck": "tsc",
32-
"postinstall": "node getDocs.js",
32+
"postinstall": "node getDocs.js && node populateStds.js && node populateCerts.js",
3333
"test": "echo \"Error: no test specified\" && exit 1",
3434
"lint:md": "markdownlint-cli2 \"**/*.md\"",
3535
"fix:md": "markdownlint-cli2-fix \"**/*.md\"",
@@ -57,7 +57,8 @@
5757
"prettier": "^2.8.4",
5858
"prism-react-renderer": "^1.3.5",
5959
"react": "^17.0.2",
60-
"react-dom": "^17.0.2"
60+
"react-dom": "^17.0.2",
61+
"yaml": "^2.3.4"
6162
},
6263
"devDependencies": {
6364
"@docusaurus/eslint-plugin": "^2.4.3",

populateCerts.js

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
const fs = require('fs')
2+
const YAML = require('yaml')
3+
4+
// how many outdated versions of any scope to include
5+
const MAX_OLD = 1
6+
7+
const filenames = fs
8+
.readdirSync('standards/')
9+
.filter((fn) => fn.startsWith('scs-') && fn.endsWith('.yaml'))
10+
11+
const scopes = filenames.map((filename) => {
12+
return {
13+
...YAML.parseDocument(fs.readFileSync(`standards/${filename}`, 'utf8')).toJSON(),
14+
filename,
15+
id: filename.substring(0, filename.length - 5),
16+
}
17+
})
18+
19+
const today = new Date().toISOString().slice(0, 10)
20+
21+
const sidebarItems = scopes.map((scope) => {
22+
const matrix = {}
23+
const versionsShown = {}
24+
var numOld = 0
25+
// sort in descending order, so we get the MAX_OLD most recent obsolete versions
26+
scope.versions.sort((a, b) => b.version.localeCompare(a.version));
27+
scope.versions.forEach((version) => {
28+
version.isStable = version.stabilized_at !== undefined && version.stabilized_at <= today
29+
version.isObsolete = version.obsoleted_at !== undefined && version.obsoleted_at < today
30+
version.isEffective = version.isStable && !version.isObsolete
31+
version.isPreview = version.stabilized_at === undefined || today < version.stabilized_at
32+
if (!version.isEffective && !version.isPreview) {
33+
numOld += 1
34+
if (numOld > MAX_OLD) return
35+
}
36+
version.state = (
37+
version.stabilized_at === undefined ? 'Draft' :
38+
version.isEffective ? 'Effective' :
39+
version.isObsolete ? 'Deprecated' :
40+
'Stable'
41+
)
42+
if (version.standards === undefined) return
43+
versionsShown[version.version] = version
44+
version.standards.forEach((standard) => {
45+
const components = standard.url.split('/')
46+
const filename = components[components.length - 1]
47+
// first, sensible (but not pretty) defaults
48+
var key = standard.url
49+
var name = standard.name
50+
var ver = '✓'
51+
var url = standard.url
52+
if (filename.startsWith('scs-') && filename.endsWith('.md')) {
53+
// special case for internal standards
54+
const components2 = filename.split('-')
55+
key = `scs-${components2[1]}`
56+
name = `${key}: ${name}`
57+
ver = components2[2]
58+
url = `/standards/${filename.substring(0, filename.length - 3)}`
59+
} else {
60+
// special case mainly for OpenStack Powered Compute, but anything ending in 'vXYZ'
61+
const components2 = name.split(' ')
62+
const v = components2.splice(components2.length - 1)
63+
if (v[0].startsWith('v')) {
64+
key = components2.join(' ')
65+
name = key
66+
ver = v[0]
67+
}
68+
}
69+
if (matrix[key] === undefined) {
70+
matrix[key] = {name, columns: {}}
71+
}
72+
matrix[key].columns[version.version] = {
73+
version: ver,
74+
url,
75+
}
76+
})
77+
})
78+
79+
const rows = Object.values(matrix)
80+
const columns = Object.keys(versionsShown)
81+
rows.sort((a, b) => a.name.localeCompare(b.name));
82+
columns.sort((a, b) => a.localeCompare(b));
83+
84+
lines = [`# ${scope.name}
85+
86+
Note that the state _Stable_ is shown here if _stabilized at_ is in the future, whereas _Effective_ is shown here if _stabilized at_ is in the past and _deprecated at_ is unset or in the future.
87+
`]
88+
lines.push('| Scope versions -> | ' + columns.join(' | ') + ' |')
89+
lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
90+
lines.push('| State | ' + columns.map((c) => versionsShown[c].state).join(' | ') + ' |')
91+
lines.push('| Stabilized at | ' + columns.map((c) => versionsShown[c].stabilized_at || '').join(' | ') + ' |')
92+
lines.push('| Obsoleted at | ' + columns.map((c) => versionsShown[c].obsoleted_at || '').join(' | ') + ' |')
93+
// md doesn't allow intermediate header rows
94+
// lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
95+
lines.push('| **Standards** | ' + columns.map((c) => ' '.repeat(c.length)).join(' | ') + ' |')
96+
// md doesn't allow intermediate header rows
97+
// lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
98+
rows.forEach((row) => {
99+
lines.push(`| ${row.name} | ` + columns.map((c) => row.columns[c]).map((col) => {
100+
if (col === undefined) {
101+
// this version of the cert does not include this standard
102+
return ''
103+
}
104+
return `[${col.version}](${col.url})`
105+
}).join(' | ') + ' |')
106+
})
107+
lines.push('') // file should end with a single newline character
108+
fs.writeFileSync(`standards/${scope.id}.md`, lines.join('\n'), 'utf8')
109+
110+
const state = columns.filter((c) => versionsShown[c].isEffective).length ? '📜' : '✏️'
111+
return {
112+
type: 'doc',
113+
label: scope.name,
114+
id: scope.id,
115+
}
116+
})
117+
118+
var newSidebars = `module.exports = ${JSON.stringify(sidebarItems, null, ' ')}`
119+
fs.writeFileSync('./sidebarsCertificationItems.js', newSidebars, 'utf8')

populateStds.js

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
const fs = require('fs')
2+
const YAML = require('yaml')
3+
4+
const intro = `# Overview
5+
6+
Standards are the core deliverable of SCS. By standardizing the open source software components of a cloud computing stack, their versions, how they are to be configured, deployed and utilized, SCS guarantees the reproducibility of a certain behavior of this technology.
7+
8+
SCS standards are discussed, developed and maintained in the community by the corresponding teams (see Track in the table below), which naturally include existing users of SCS.`
9+
const trackIntros = {
10+
'Global': 'This track encompasses the foundational standards that guide the overall structure, documentation, and general topics related to the Sovereign Cloud Stack. It serves as the core framework, ensuring consistency, clarity, and comprehensibility across all aspects of the cloud stack, fostering an environment where information is easily accessible and understood.',
11+
'IaaS': 'The IaaS Layer Standards track focuses on the protocols, guidelines, and specifications that govern the infrastructure as a service layer. This encompasses standards for virtual machines, storage, networking, and other foundational resources, ensuring seamless, efficient, and secure operation, interoperability, and management of the underlying cloud infrastructure.',
12+
'KaaS': 'Standards in this track are concerned with Kubernetes as a Service layer, outlining norms and best practices for deploying, managing, and operating Kubernetes clusters. These standards aim to ensure that the orchestration of containers is streamlined, secure, and compatible across various cloud environments and platforms.',
13+
'IAM': 'This track revolves around Identity and Access Management (IAM) standards, providing guidelines for ensuring secure and efficient user authentication, authorization, and administration. It addresses issues related to user identity, permissions, roles, and policies, aiming to safeguard and streamline access to cloud resources and services.',
14+
'Ops': 'Operational Tooling Standards cover the protocols and guidelines associated with tools and utilities used for monitoring, management, and maintenance of the cloud environment. This includes standards for status pages, alerts, logs, and other operational tools, aiming to optimize the reliability, performance, and security of cloud services and resources.',
15+
}
16+
const headerLegend = '*Legend to the column headings: Draft, Stable (but not effective), Effective, Deprecated (and no longer effective).'
17+
18+
var filenames = fs
19+
.readdirSync('standards/')
20+
.filter((fn) => fn.startsWith('scs-') && fn.endsWith('.md') && !fn.startsWith('scs-X'))
21+
22+
keys = ['title', 'type', 'status', 'track', 'stabilized_at', 'obsoleted_at', 'replaces', 'authors', 'state']
23+
24+
// use the ISO string, because so do the standard documents, and we can use string comparison with ISO dates
25+
today = new Date().toISOString().slice(0, 10)
26+
27+
// collect all the information sorted into a track/adr-id/version hierarchy
28+
tracks = {}
29+
filenames.forEach((filename) => {
30+
var components = filename.split('-')
31+
var obj = {
32+
...YAML.parseDocument(fs.readFileSync(`standards/${filename}`, 'utf8')).toJSON(),
33+
filename,
34+
id: filename.substring(0, filename.length - 3),
35+
adrId: components[1],
36+
version: components[2],
37+
}
38+
obj.isStable = obj.stabilized_at !== undefined && obj.stabilized_at <= today
39+
obj.isObsolete = obj.obsoleted_at !== undefined && obj.obsoleted_at <= today
40+
obj.isEffective = obj.isStable && !obj.isObsolete
41+
var track = obj.track
42+
if (track === undefined) return
43+
if (tracks[track] === undefined) tracks[track] = {}
44+
var standards = tracks[track]
45+
if (standards[obj.adrId] === undefined) standards[obj.adrId] = {versions: []}
46+
standards[obj.adrId].versions.push(obj)
47+
})
48+
49+
function readPrefixLines(fn) {
50+
var lines = []
51+
if (fs.existsSync(fn)) {
52+
lines = fs.readFileSync(fn, 'utf8').split('\n')
53+
var tableIdx = lines.findIndex((line) => line.trim().startsWith('|'))
54+
if (tableIdx >= 0) {
55+
lines.splice(tableIdx)
56+
}
57+
} else console.log(`WARNING: file ${fn} not found`)
58+
return lines
59+
}
60+
61+
function mkLinkList(versions) {
62+
var links = versions.map((v) => `[${v.version}](/standards/${v.id})`)
63+
return links.join(', ')
64+
}
65+
66+
// walk down the hierarchy, building adr overview pages, track overview pages, and total overview page
67+
// as well as the new sidebar
68+
sidebarItems = []
69+
var lines = readPrefixLines('standards/standards/overview.md')
70+
if (!lines.length) lines.push(`${intro}
71+
72+
${headerLegend}
73+
`)
74+
lines.push('| Standard | Track | Description | Draft | Stable* | Effective | Deprecated* |')
75+
lines.push('| --------- | ------ | ------------ | ----- | ------- | --------- | ----------- |')
76+
Object.entries(tracks).forEach((trackEntry) => {
77+
var track = trackEntry[0]
78+
var trackPath = `standards/${track.toLowerCase()}`
79+
fs.mkdirSync(trackPath, { recursive: true })
80+
var trackItem = {
81+
type: 'category',
82+
label: track,
83+
link: {
84+
type: 'doc',
85+
id: `${track.toLowerCase()}/index`,
86+
},
87+
items: [],
88+
}
89+
sidebarItems.push(trackItem)
90+
var tlines = readPrefixLines(`standards/${track.toLowerCase()}/index.md`)
91+
if (!tlines.length) {
92+
tlines.push(`# ${track} Standards
93+
94+
${trackIntros[track]}
95+
96+
${headerLegend}
97+
`)
98+
}
99+
tlines.push('| Standard | Description | Draft | Stable* | Effective | Deprecated* |')
100+
tlines.push('| --------- | ------------ | ----- | ------- | --------- | ----------- |')
101+
Object.entries(trackEntry[1]).forEach((standardEntry) => {
102+
var versions = standardEntry[1].versions
103+
// unfortunately, some standards are obsolete without being stable
104+
var draftVersions = versions.filter((v) => v.stabilized_at === undefined && v.obsoleted_at === undefined)
105+
var stableVersions = versions.filter((v) => v.stabilized_at !== undefined && !v.isEffective)
106+
var effectiveVersions = versions.filter((v) => v.isEffective)
107+
var deprecatedVersions = versions.filter((v) => v.isObsolete)
108+
var ref = versions[versions.length - 1]
109+
if (effectiveVersions.length) {
110+
ref = effectiveVersions[effectiveVersions.length - 1]
111+
}
112+
var adrId = standardEntry[0]
113+
var standardItem = {
114+
type: 'category',
115+
label: `scs-${adrId}`,
116+
link: {
117+
type: 'doc',
118+
id: `${track.toLowerCase()}/scs-${adrId}`,
119+
},
120+
items: [],
121+
}
122+
trackItem.items.push(standardItem)
123+
var slines = readPrefixLines(`standards/${track.toLowerCase()}/scs-${adrId}.md`)
124+
if (!slines.length) {
125+
slines.push(`# scs-${adrId}: ${ref.title}\n`)
126+
if (ref.description !== undefined) {
127+
slines.push(ref.description)
128+
}
129+
}
130+
slines.push('| Version | Type | State | stabilized | obsoleted |')
131+
slines.push('| -------- | ----- | ------- | ---------- | --------- |')
132+
var link = `[scs-${adrId}](/standards/${track.toLowerCase()}/scs-${adrId})`
133+
var versionList = `${mkLinkList(draftVersions) || '-'} | ${mkLinkList(stableVersions) || '-'} | ${mkLinkList(effectiveVersions) || '-'} | ${mkLinkList(deprecatedVersions) || '-'}`
134+
lines.push(`| ${link} | ${track} | ${ref.title} | ${versionList} |`)
135+
tlines.push(`| ${link} | ${ref.title} | ${versionList} |`)
136+
standardEntry[1].versions.forEach((obj) => {
137+
var versionItem = {
138+
type: 'doc',
139+
label: obj.version.toUpperCase(),
140+
id: obj.id,
141+
}
142+
standardItem.items.push(versionItem)
143+
slines.push(`| [scs-${adrId}-${obj.version}](/standards/${obj.id}) | ${obj.type} | ${obj.status || obj.state} | ${obj.stabilized_at || '-'} | ${obj.obsoleted_at || '-'} |`)
144+
})
145+
slines.push('') // file should end with a single newline character
146+
fs.writeFileSync(`${trackPath}/scs-${adrId}.md`, slines.join('\n'), 'utf8')
147+
})
148+
tlines.push('') // file should end with a single newline character
149+
fs.writeFileSync(`${trackPath}/index.md`, tlines.join('\n'), 'utf8')
150+
})
151+
lines.push('') // file should end with a single newline character
152+
fs.writeFileSync(`standards/standards/overview.md`, lines.join('\n'), 'utf8')
153+
154+
var newSidebars = `module.exports = ${JSON.stringify(sidebarItems, null, ' ')}`
155+
fs.writeFileSync('./sidebarsStandardsItems.js', newSidebars, 'utf8')

0 commit comments

Comments
 (0)