GetDKAN#4347: Move and remove elements from JSON arrays in form - mer…

…ge 2.x

.circleci/config.yml

@@ -29,7 +29,7 @@ commands:
         default: '8.2'
         type: string
       addon_branch:
-        description: 'Repo branch name for the dkan-ddev-addon you want to test against.'
+        description: 'Repo branch name for the DKAN DDev add-on you want to test against.'
         default: 'main'
         type: string
       dkan_recommended_branch:
@@ -182,7 +182,7 @@ jobs:
 
   cypress:
     machine:
-      image: default
+      image: ubuntu-2204:current
     parallelism: 4
     parameters:
       php_version:
@@ -230,39 +230,39 @@ workflows:
     jobs:
       - cypress:
           name: install_test_cypress
-          dkan_recommended_branch: '10.2.x-dev'
+          dkan_recommended_branch: '10.3.x-dev'
       - phpunit:
-          name: 'Install target (Drupal 10.2, PHP 8.3)'
+          name: 'Install target (Drupal 10.3, PHP 8.3)'
           report_coverage: true
           matrix:
             parameters:
-              dkan_recommended_branch: ['10.2.x-dev']
+              dkan_recommended_branch: ['10.3.x-dev']
               php_version: [ '8.3' ]
       - phpunit:
           matrix:
             parameters:
-              dkan_recommended_branch: [ '10.3.x-dev']
+              dkan_recommended_branch: [ '10.4.x-dev']
               php_version: [ '8.3' ]
       - phpunit:
           matrix:
             parameters:
-              dkan_recommended_branch: [ '10.2.x-dev']
+              dkan_recommended_branch: [ '10.3.x-dev']
               php_version: [ '8.1', '8.2' ]
       - phpunit:
           matrix:
             parameters:
-              dkan_recommended_branch: [ '10.1.x-dev']
-              php_version: [ '8.2' ]
+              dkan_recommended_branch: [ '10.2.x-dev']
+              php_version: [ '8.3' ]
   upgrade_and_test:
     jobs:
       - cypress:
           name: upgrade_test_cypress
           upgrade: true
           dkan_recommended_branch: '10.2.x-dev'
       - phpunit:
-          name: 'Upgrade target (Drupal 10.2, PHP 8.2)'
+          name: 'Upgrade target (Drupal 10.3, PHP 8.3)'
           upgrade: true
           matrix:
             parameters:
-              dkan_recommended_branch: [ '10.2.x-dev']
-              php_version: [ '8.2' ]
+              dkan_recommended_branch: [ '10.3.x-dev']
+              php_version: [ '8.3' ]

.codeclimate.yml

@@ -1,4 +1,4 @@
-version: 2
+version: "2"
 plugins:
   phpcodesniffer:
     enabled: true
@@ -14,6 +14,8 @@ plugins:
         enabled: false
       Drupal Commenting Deprecated DeprecatedWrongSeeUrlFormat:
         enabled: false
+      Drupal Commenting VariableComment MissingVar:
+        enabled: false
 
 exclude_patterns:
   - "cypress/"

README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/a02bf3362b94749579a1/maintainability)](https://codeclimate.com/github/GetDKAN/dkan/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/a02bf3362b94749579a1/test_coverage)](https://codeclimate.com/github/GetDKAN/dkan/test_coverage)
 [![GPL license](https://img.shields.io/badge/License-GPL(>=2)-blue.svg)](http://www.gnu.org/licenses/gpl.html)
-
+[![DPG Badge](https://img.shields.io/badge/Verified-DPG-3333AB?logo=data:image/svg%2bxml;base64,PHN2ZyB3aWR0aD0iMzEiIGhlaWdodD0iMzMiIHZpZXdCb3g9IjAgMCAzMSAzMyIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0LjIwMDggMjEuMzY3OEwxMC4xNzM2IDE4LjAxMjRMMTEuNTIxOSAxNi40MDAzTDEzLjk5MjggMTguNDU5TDE5LjYyNjkgMTIuMjExMUwyMS4xOTA5IDEzLjYxNkwxNC4yMDA4IDIxLjM2NzhaTTI0LjYyNDEgOS4zNTEyN0wyNC44MDcxIDMuMDcyOTdMMTguODgxIDUuMTg2NjJMMTUuMzMxNCAtMi4zMzA4MmUtMDVMMTEuNzgyMSA1LjE4NjYyTDUuODU2MDEgMy4wNzI5N0w2LjAzOTA2IDkuMzUxMjdMMCAxMS4xMTc3TDMuODQ1MjEgMTYuMDg5NUwwIDIxLjA2MTJMNi4wMzkwNiAyMi44Mjc3TDUuODU2MDEgMjkuMTA2TDExLjc4MjEgMjYuOTkyM0wxNS4zMzE0IDMyLjE3OUwxOC44ODEgMjYuOTkyM0wyNC44MDcxIDI5LjEwNkwyNC42MjQxIDIyLjgyNzdMMzAuNjYzMSAyMS4wNjEyTDI2LjgxNzYgMTYuMDg5NUwzMC42NjMxIDExLjExNzdMMjQuNjI0MSA5LjM1MTI3WiIgZmlsbD0id2hpdGUiLz4KPC9zdmc+Cg==)](https://www.digitalpublicgoods.net/r/dkan)
 ---
 
 [![Getting Started with DKAN](https://img.youtube.com/vi/SnA22Lb6r_M/0.jpg)](https://youtu.be/SnA22Lb6r_M?si=WqU-FKGb1iGhc5Vz)
@@ -44,7 +44,7 @@ DKAN and related modules are freely-available under the ["GNU General Public Lic
 
 ## Security
 
-If you have found a vunerability in DKAN, please report this by e-mailing [email protected].
+If you have found a vulnerability in DKAN, please report this by e-mailing [email protected].
 
 ---
 

composer.json

@@ -15,17 +15,17 @@
         "drupal/moderated_content_bulk_publish": "~2.0.20",
         "drupal/search_api": "^1.15",
         "drupal/select2": "^1.13",
-        "drupal/select_or_other": "^4.1.0",
+        "drupal/select_or_other": "^4.2.0",
         "drupal/views_bulk_operations": "^4.0",
         "ezyang/htmlpurifier": "^4.11",
         "fmizzell/maquina": "^1.1.1",
         "fylax/forceutf8": "^3.0",
-        "getdkan/contracts": "^1.1.3",
-        "getdkan/csv-parser": "^1.3.2",
-        "getdkan/file-fetcher": "^5.0.4",
-        "getdkan/harvest": "^1.0.4",
+        "getdkan/contracts": "^1.2.0",
+        "getdkan/csv-parser": "^1.3.3",
+        "getdkan/file-fetcher": "^5.0.5",
+        "getdkan/harvest": "^1.0.5",
         "getdkan/pdlt": "^0.1.7",
-        "getdkan/procrastinator": "^5.0.2",
+        "getdkan/procrastinator": "^5.0.3",
         "getdkan/rooted-json-data": "^0.2.2",
         "guzzlehttp/guzzle": "^6.5.8 || ^7.4.5",
         "ilbee/csv-response": "^1.2.0",

cypress/integration/09_admin_links.spec.js

@@ -41,6 +41,7 @@ context('Administration pages', () => {
     })
     cy.visit(baseurl + "/admin/dkan/datastore")
     cy.get('label[for="edit-rows-limit"]').should('have.text', 'Rows limit')
+    cy.get('label[for="edit-response-stream-max-age"]').should('have.text', 'Response Stream Max-Age')
   })
 
   it('Admin can access the Datastore import status dashboard.', () => {

docs/requirements.txt

@@ -2,6 +2,7 @@ myst-parser>=2.0.0
 sphinx==7.2.6
 sphinx-rtd-theme>=1.2.2
 sphinxcontrib-httpdomain>=1.8.1
+sphinxcontrib.mermaid>=1.0.0
 sphinxcontrib-phpdomain>=0.11.1
 pygments>=2.14
 sphinx-prompt>=1.8.0

docs/source/components/index.rst

@@ -8,3 +8,4 @@ Components
    dkan_datastore
    dkan_harvest
    dkan_api
+   json_form_widget

docs/source/components/json_form_widget.rst

@@ -0,0 +1,2 @@
+.. include:: ../../../modules/json_form_widget/README.md
+    :parser: myst_parser.sphinx_

docs/source/conf.py

@@ -36,6 +36,7 @@
   'myst_parser',
   'sphinx.ext.autosectionlabel',
   'sphinxcontrib.httpdomain',
+  'sphinxcontrib.mermaid',
   'sphinx_rtd_theme',
   'sphinx_prompt'
 ]
@@ -76,6 +77,7 @@
     '.md': 'markdown',
 }
 source_parsers = {'.md': 'myst_parser.sphinx_'}
+myst_fence_as_directive = ["mermaid"]
 
 autosectionlabel_prefix_document = True
 autosectionlabel_maxdepth=None

docs/source/contributing/documentation.rst

@@ -2,8 +2,8 @@ Contributing to DKAN documentation
 ==================================
 
 What follows is a style guide for the DKAN documentation. Use it both to follow the conventions used throughout the site,
-and for your own contributions. DKAN's documentaion is written in a combination of `Markdown <https://daringfireball.net/projects/markdown>`_
-and `ReStructuredText (RST) <http://www.sphinx-doc.org/en/stable/rest.html>`_, and built with `Sphynx <http://www.sphinx-doc.org/en/stable/index.html>`_.
+and for your own contributions. DKAN's documentation is written in a combination of `Markdown <https://daringfireball.net/projects/markdown>`_
+and `ReStructuredText (RST) <http://www.sphinx-doc.org/en/stable/rest.html>`_, and built with `Sphinx <http://www.sphinx-doc.org/en/stable/index.html>`_.
 The docs live in the `/docs/source` folder of the `DKAN Project <https://github.com/GetDKAN/dkan>`_; to suggest modifications,
 submit a pull request as you would for any suggested code change.
 

docs/source/drush_commands.rst

@@ -198,7 +198,7 @@ dkan:harvest:register
 dkan:harvest:revert
 --------------------
 
-    Revert a harvest. Removes harvested entities and unpublishes orhpaned
+    Revert a harvest. Removes harvested entities and unpublishes orphaned
     keywords, themes, and distributions. The harvest plan will remain and can
     be run again to generate the datasets after any issues have been resolved.
 

docs/source/installation/index.rst

@@ -18,7 +18,7 @@ This will ensure you have the same setup as DKAN's core developers, and that you
 Requirements
 ------------
 
-DKAN is based on `Drupal software <https://www.drupal.org/docs/getting-started/system-requirements>`_ and -- generally -- runs anywhere Drupal is supported. For the most common setup, we reccomend:
+DKAN is based on `Drupal software <https://www.drupal.org/docs/getting-started/system-requirements>`_ and -- generally -- runs anywhere Drupal is supported. For the most common setup, we recommend:
 
 -  Drupal 10+
 -  MySQL: minimum version 5.7.8+ with PDO

docs/source/introduction/glossary.rst

@@ -17,7 +17,7 @@ Glossary
     This refers to the actual data, usually provided as a file (csv, xls, xlsx, dat, zip, tar, pdf, etc.) but may also be a URL that provides access to the data. CSV is one of the most widely used data file formats and can be opened and edited by almost any simple text editor as well as Microsoft Excel, Apple Numbers, or Google Spreadsheets.
 
   Dataset
-    A dataset is an identifiable collection of structured data objects unified by some criteria (authorship, subject, scope, spatial or temporal extent…) this unifiying criteria is called metadata. In DKAN, the term **dataset** refers to the metadata plus the data resource(s). A dataset can have multiple data resources and these are listed under the metadata property called **distribution**.
+    A dataset is an identifiable collection of structured data objects unified by some criteria (authorship, subject, scope, spatial or temporal extent…) this unifying criteria is called metadata. In DKAN, the term **dataset** refers to the metadata plus the data resource(s). A dataset can have multiple data resources and these are listed under the metadata property called **distribution**.
 
   Datastore
     A datastore is the data resource stored in a database. DKAN will import data from a UTF-8 encoded csv file into a database table and provide an API endpoint from which other applications can run queries on that data.

docs/source/user-guide/guide_custom_schemas.rst

@@ -19,14 +19,22 @@ Then make any modifications necessary to the `dataset.json` file inside the `col
 
   }
 
-You can remove metadata fields with the exception of the *distribution* property, this one must remain present in the dataset.json file.
+.. note::
 
-Note that even if you are only changing the dataset.json schema, it is important to copy ALL of the schema files as DKAN will be expecting all of the
-schema files to be in the same location.
+  Note that even if you are only changing the dataset.json schema, it is important to copy ALL of the schema files as DKAN will be expecting all of the
+  schema files to be in the same location.
 
 .. warning::
 
-  Warning: The schema is actively used by the catalog to verify the validity of the data.
+  You can change or remove metadata fields from the dataset schema without causing issues, with the exception of the *distribution* property. If there is no array of objects called ``distribution``, each containing a ``downloadURL`` field, the :doc:`datastore <../components/dkan_datastore>` will not detect resource for import. This may or may not be an issue, depending on your use case. We hope to improve this in the future so that the datastore is not dependent on a particular metadata schema.
+
+.. warning::
+
+  The schema is actively used by the catalog to verify the validity of the data.
   Making changes to the schema after data is present in the catalog should be done with care
   as non-backward-compatible changes to the schema could cause issues.
   Look at ``Drupal::metastore::SchemaRetriever::findSchemaDirectory()`` for context.
+
+.. note::
+
+  For more information on how forms are generated from schema files, see :doc:`JSON Form Widget <../components/json_form_widget>`.

docs/source/user-guide/guide_data_dictionaries.rst

@@ -96,7 +96,7 @@ This property is important for fields where you need to specify the format of th
 If your date values are not in ISO8601 format, use this property to define the format being used so that the data will import into the datastore correctly. Month and day values must be zero-padded. Follow the date formatting syntax of C / Python `strftime <http://strftime.org/>`_ to determine the pattern to use in your format property. For example, if your dates are in mm/dd/YYYY format, use "format": "%m/%d/%Y".
 
 .. Note::
-  The "Download full dataset (CSV)" button will download the original source file. The "Download filtered data (CSV)" button will generate a new file, using the data dictioinary title values (if present) for the column headings, otherwise the column headings from the source file will be used.
+  The "Download full dataset (CSV)" button will download the original source file. The "Download filtered data (CSV)" button will generate a new file, using the data dictionary title values (if present) for the column headings, otherwise the column headings from the source file will be used.
 
 How to create a data dictionary
 -------------------------------
@@ -230,7 +230,7 @@ To set the data dictionary mode to **sitewide**:
 4. Click "Save configuration".
 
 .. image:: images/dictionary-settings.png
-  :alt: Data dictionay settings admin page, with select input for "Dictionary Mode" set to "Sitewide" and text
+  :alt: Data dictionary settings admin page, with select input for "Dictionary Mode" set to "Sitewide" and text
         input for Sitewide Dictionary ID containing the identifier 7fd6bb1f-2752-54de-9a33-81ce2ea0feb2.
 
 
@@ -249,7 +249,7 @@ To set the data dictionary mode to **distribution reference**:
 .. NOTE::
    Assigning data dictionaries to datasets can be done on the dataset form. Enter the API endpoint of the data dictionary into the "Data Dictionary" field of the distribution section. Set the "Data Dictionary Type" field to *application/vnd.tableschema+json*.
 
-Or, use the API to link a new dataset to the data dictionay.
+Or, use the API to link a new dataset to the data dictionary.
 Look closely at the distribution property in the example below, this is using the data dictionary uuid from the example above.
 
 .. http:post:: /api/1/metastore/schemas/dataset/items

docs/source/user-guide/guide_harvest.rst

@@ -50,7 +50,7 @@ If you plan to maintain the datasets with the harvest process then do not edit t
 Edit the metadata in the data.json file and re-run the harvest to update the datasets.
 
 If you are using the harvest to simply bulk generate datasets, and want to allow data publishers to update the datasets as needed
-via the UI or API, delete the data.json file to prevent overwritting any changes. After the 2.16.13 release, you could also
+via the UI or API, delete the data.json file to prevent overwriting any changes. After the 2.16.13 release, you could also
 deregister the harvest.
 
 Register a harvest
@@ -73,7 +73,7 @@ Register a harvest
 Run the harvest
 ---------------
   Once you have registered a harvest source, run the import, passing in
-  the identifier as an arguement
+  the identifier as an argument
 
   .. prompt:: bash $
 

docs/source/user-guide/guide_local_files.rst

@@ -3,7 +3,7 @@ Importing very large files
 
 Very large datasets can present challenges for the datastore import process. What "very large" means will depend on your server infrastructure, but it's safe to say that when dealing with multi-gigabyte CSV files you may start to encounter broken file transfers or timeouts.
 
-Because DKAN will make a local copy of a remote file in order to import it, it can help to prepare the import by transfering the files to the web server manually (or with shell scripts). Then you can instruct DKAN to use that already-existing local copy for processing into the database.
+Because DKAN will make a local copy of a remote file in order to import it, it can help to prepare the import by transferring the files to the web server manually (or with shell scripts). Then you can instruct DKAN to use that already-existing local copy for processing into the database.
 
 These instructions assume you've already created a dataset (through the the UI, a harvest or the API) that contains a distribution pointing to a remote CSV file, and that this CSV file has not yet been imported to the datastore. We'll need the ID of the existing dataset.
 

docs/source/user-guide/guide_metastore_properties.rst

@@ -4,7 +4,7 @@ How to add or remove entity generation for schema sub properties
 When you create a dataset, additional data nodes will be created for specific sub-elements
 of the dataset as well, the default properties are: publisher, theme, keywords, and
 distribution. These data nodes will provide unique reference ids for the sub-elements and
-can be accesssed via an API endpoint. Learn about the `API <https://demo.getdkan.org/api>`_.
+can be accessed via an API endpoint. Learn about the `API <https://demo.getdkan.org/api>`_.
 
 You can customize which sub-elements generate additional data nodes here ``admin/dkan/properties``.
 

modules/common/src/Commands/JobStoreCommands.php

@@ -15,8 +15,6 @@ class JobStoreCommands extends DrushCommands {
 
   /**
    * Database connection.
-   *
-   * @var \Drupal\Core\Database\Connection
    */
   protected Connection $connection;
 

modules/common/src/DataResource.php

@@ -135,9 +135,9 @@ public function __construct($file_path, $mimeType, $perspective = self::DEFAULT_
    * @return \Drupal\common\DataResource
    *   DataResource object.
    *
-   * @deprecated Use DataResource::createFromEntity() instead.
-   *
-   * @see self::createFromEntity()
+   * @deprecated in dkan:8.x-2.17 and is removed from dkan:8.x-2.21. Use
+   *   DataResource::createFromEntity() instead.
+   * @see https://github.com/GetDKAN/dkan/pull/4027
    */
   public static function createFromRecord(object $record): DataResource {
     $resource = new static($record->filePath, $record->mimeType, $record->perspective);
@@ -236,6 +236,11 @@ public function changeMimeType($newMimeType) {
    *
    * @return \Drupal\datastore\DatastoreResource
    *   Datastore Resource.
+   *
+   * @deprecated in dkan:8.x-2.20 and is removed from dkan:8.x-2.21. Use storage
+   *   classes like DatabaseTable::getTableName() to determine correct table
+   *   names, and pass true to ::getFilePath to get the resolved URL.
+   * @see https://github.com/GetDKAN/dkan/pull/4372
    */
   public function getDatastoreResource(): DatastoreResource {
     return new DatastoreResource(
@@ -253,10 +258,16 @@ public function getIdentifier() {
   }
 
   /**
-   * Getter.
+   * Get the resource's filepath.
+   *
+   * @param bool|null $resolve
+   *   Whether to resolve the URL host tokens in the file path.
+   *
+   * @return string
+   *   The file path.
    */
-  public function getFilePath() {
-    return $this->filePath;
+  public function getFilePath(?bool $resolve = FALSE):string {
+    return $resolve ? UrlHostTokenResolver::resolve($this->getFilePath()) : $this->filePath;
   }
 
   /**
@@ -324,6 +335,11 @@ public function getUniqueIdentifierNoPerspective(): string {
 
   /**
    * Retrieve datastore table name for resource.
+   *
+   * @deprecated in dkan:8.x-2.20 and is removed from dkan:8.x-2.21. Use storage
+   *  classes like DatabaseTable::getTableName() to determine correct table
+   *  names.
+   * @see https://github.com/GetDKAN/dkan/pull/4372
    */
   public function getTableName() {
     return 'datastore_' . md5($this->getUniqueIdentifier());
@@ -333,7 +349,7 @@ public function getTableName() {
    * {@inheritDoc}
    */
   #[\ReturnTypeWillChange]
-  public function jsonSerialize() {
+  public function jsonSerialize(): mixed {
     return $this->serialize();
   }
 

modules/common/src/DkanApiDocsGenerator.php

@@ -14,15 +14,11 @@ class DkanApiDocsGenerator {
 
   /**
    * Docs manager.
-   *
-   * @var \Drupal\common\Plugin\DkanApiDocsPluginManager
    */
   protected DkanApiDocsPluginManager $docManager;
 
   /**
    * Site settings.
-   *
-   * @var \Drupal\Core\Site\Settings
    */
   protected Settings $settings;
 

modules/common/src/Events/Event.php

@@ -29,8 +29,6 @@ class Event extends DrupalEvent {
 
   /**
    * Exception.
-   *
-   * @var \Exception|null
    */
   private ?\Exception $exception;