Add ability to show files per build and whitelisting

.gitignore

@@ -11,3 +11,4 @@ CMakeFiles/
 cmake_install.cmake
 build/
 *.cbp
+.vscode/settings.json

CMakeLists.txt

@@ -37,7 +37,7 @@ file(GLOB SRC_FILES
   src/downloader.cpp
   src/progressbar.cpp
   src/util.cpp
-  src/blacklist.cpp
+  src/filelist.cpp
   src/gamefile.cpp
   src/gamedetails.cpp
   src/galaxyapi.cpp
@@ -149,11 +149,7 @@ if(USE_QT_GUI)
   )
 endif(USE_QT_GUI)
 
-if(Qt6_FOUND)
-  set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17)
-else()
-  set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 11)
-endif(Qt6_FOUND)
+set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17)
 
 if(MSVC)
   # Force to always compile with W4

include/config.h

@@ -12,8 +12,9 @@
 #include <json/json.h>
 #include <mutex>
 #include <ctime>
+#include <optional>
 
-#include "blacklist.h"
+#include "filelist.h"
 
 struct DirectoryConfig
 {
@@ -281,6 +282,7 @@ class Config
         // File paths
         std::string sConfigFilePath;
         std::string sBlacklistFilePath;
+        std::string sWhitelistFilePath;
         std::string sIgnorelistFilePath;
         std::string sGameHasDLCListFilePath;
         std::string sReportFilePath;
@@ -304,9 +306,10 @@ class Config
         std::string sPassword;
 
         // Lists
-        Blacklist blacklist;
-        Blacklist ignorelist;
-        Blacklist gamehasdlc;
+        std::optional<Filelist> blacklist;
+        std::optional<Filelist> ignorelist;
+        std::optional<Filelist> gamehasdlc;
+        std::optional<Filelist> whitelist;
 
         // Cloud save options
         std::vector<std::string> cloudWhiteList;

include/downloader.h

@@ -125,10 +125,12 @@ class Downloader
         void galaxyInstallGame(const std::string& product_id, int build_index = -1, const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64);
         void galaxyInstallGameById(const std::string& product_id, int build_index = -1, const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64);
         void galaxyShowBuilds(const std::string& product_id, int build_index = -1);
+        void galaxyShowBuildFiles(const std::string& product_id, int build_index = -1);
         void galaxyShowCloudSaves(const std::string& product_id, int build_index = -1);
         void galaxyShowLocalCloudSaves(const std::string& product_id, int build_index = -1);
         void galaxyShowLocalCloudSavesById(const std::string& product_id, int build_index = -1);
         void galaxyShowBuildsById(const std::string& product_id, int build_index = -1);
+        void galaxyShowBuildFilesById(const std::string& product_id, int build_index = -1);
         void galaxyShowCloudSavesById(const std::string& product_id, int build_index = -1);
     protected:
     private:
@@ -153,9 +155,9 @@ class Downloader
         static std::string getChangelogFromJSON(const Json::Value& json);
         void saveJsonFile(const std::string& json, const std::string& filepath);
         void saveChangelog(const std::string& changelog, const std::string& filepath);
-        static void processDownloadQueue(Config conf, const unsigned int& tid);
-        static void processCloudSaveDownloadQueue(Config conf, const unsigned int& tid);
-        static void processCloudSaveUploadQueue(Config conf, const unsigned int& tid);
+        static void processDownloadQueue(const Config& conf, const unsigned int& tid);
+        static void processCloudSaveDownloadQueue(const Config& conf, const unsigned int& tid);
+        static void processCloudSaveUploadQueue(const Config& conf, const unsigned int& tid);
         static int progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
         template <typename T> void printProgress(const ThreadSafeQueue<T>& download_queue);
         static void getGameDetailsThread(Config config, const unsigned int& tid);
@@ -167,10 +169,10 @@ class Downloader
         static size_t readData(void *ptr, size_t size, size_t nmemb, FILE *stream);
 
         std::vector<std::string> galaxyGetOrphanedFiles(const std::vector<galaxyDepotItem>& items, const std::string& install_path);
-        static void processGalaxyDownloadQueue(const std::string& install_path, Config conf, const unsigned int& tid);
+        static void processGalaxyDownloadQueue(const std::string& install_path, const Config& conf, const unsigned int& tid);
         void galaxyInstallGame_MojoSetupHack(const std::string& product_id);
         void galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendtoFirst = false);
-        static void processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid);
+        static void processGalaxyDownloadQueue_MojoSetupHack(const Config& conf, const unsigned int& tid);
         int mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEntry>& vFiles);
         std::string getGalaxyInstallDirectory(galaxyAPI *galaxyHandle, const Json::Value& manifest);
         bool galaxySelectProductIdHelper(const std::string& product_id, std::string& selected_product);

include/blacklist.h → include/filelist.h

@@ -4,8 +4,8 @@
  * To Public License, Version 2, as published by Sam Hocevar. See
  * http://www.wtfpl.net/ for more details. */
 
-#ifndef BLACKLIST_H__
-#define BLACKLIST_H__
+#ifndef FILELIST_H
+#define FILELIST_H
 
 #include <boost/regex.hpp>
 #include <string>
@@ -14,27 +14,26 @@
 class Config;
 class gameFile;
 
-class BlacklistItem {
+class FilelistItem {
     public:
-        unsigned int linenr; // where the blacklist item is defined in blacklist.txt
+        unsigned int linenr; // where the blacklist item is defined in black/white list.txt
         unsigned int flags;
         std::string source; // source representation of the item
         boost::regex regex;
 };
 
-class Blacklist
+class Filelist
 {
     public:
-        Blacklist() {};
+        Filelist(const std::vector<std::string>& lines);
 
-        void initialize(const std::vector<std::string>& lines);
-        bool isBlacklisted(const std::string& path);
-        bool isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory = "");
+        bool Matches(const std::string& path) const;
+        bool Matches(const std::string& path, const std::string& gamename, std::string subdirectory = "") const;
 
-        std::vector<BlacklistItem>::size_type size() const { return blacklist_.size(); }
-        bool empty() { return blacklist_.empty(); }
+        std::vector<FilelistItem>::size_type size() const { return files.size(); }
+        bool empty() const { return files.empty(); }
     private:
-        std::vector<BlacklistItem> blacklist_;
+        std::vector<FilelistItem> files;
 };
 
-#endif // BLACKLIST_H_
+#endif  // FILELIST_H

include/util.h

@@ -63,7 +63,7 @@ namespace Util
     std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
     std::string getFileHash(const std::string& filename, unsigned hash_id);
     std::string getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start = 0, off_t range_end = 0);
-    std::string getChunkHash(unsigned char* chunk, uintmax_t chunk_size, unsigned hash_id);
+    std::string getChunkHash(const char* chunk, uintmax_t chunk_size, unsigned hash_id);
     int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string());
     int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string());
     int replaceString(std::string& str, const std::string& to_replace, const std::string& replace_with);

main.cpp

@@ -11,6 +11,7 @@
 #include "galaxyapi.h"
 #include "globals.h"
 
+#include <optional>
 #include <fstream>
 #include <boost/filesystem.hpp>
 #include <boost/program_options.hpp>
@@ -58,55 +59,57 @@ int main(int argc, char *argv[])
     Globals::globalConfig.curlConf.sCookiePath = Globals::globalConfig.sConfigDirectory + "/cookies.txt";
     Globals::globalConfig.sConfigFilePath = Globals::globalConfig.sConfigDirectory + "/config.cfg";
     Globals::globalConfig.sBlacklistFilePath = Globals::globalConfig.sConfigDirectory + "/blacklist.txt";
+    Globals::globalConfig.sWhitelistFilePath = Globals::globalConfig.sConfigDirectory + "/whitelist.txt";
     Globals::globalConfig.sIgnorelistFilePath = Globals::globalConfig.sConfigDirectory + "/ignorelist.txt";
     Globals::globalConfig.sGameHasDLCListFilePath = Globals::globalConfig.sConfigDirectory + "/game_has_dlc.txt";
     Globals::globalConfig.sTransformConfigFilePath = Globals::globalConfig.sConfigDirectory + "/transformations.json";
 
     Globals::galaxyConf.setFilepath(Globals::globalConfig.sConfigDirectory + "/galaxy_tokens.json");
 
     std::string sDefaultBlacklistFilePath = Globals::globalConfig.sConfigDirectory + "/blacklist.txt";
+    std::string sDefaultWhitelistFilePath = Globals::globalConfig.sConfigDirectory + "/whitelist.txt";
     std::string sDefaultIgnorelistFilePath = Globals::globalConfig.sConfigDirectory + "/ignorelist.txt";
 
     std::string priority_help_text = "Set priority by separating values with \",\"\nCombine values by separating with \"+\"";
     // Create help text for --platform option
     std::string platform_text = "Select which installers are downloaded\n";
-    for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i)
+    for (const auto& platform : GlobalConstants::PLATFORMS)
     {
-        platform_text += GlobalConstants::PLATFORMS[i].str + " = " + GlobalConstants::PLATFORMS[i].regexp + "\n";
+        platform_text += platform.str + " = " + platform.regexp + "\n";
     }
     platform_text += "All = all";
     platform_text += "\n\n" + priority_help_text;
     platform_text += "\nExample: Linux if available otherwise Windows and Mac: l,w+m";
 
     // Create help text for --galaxy-platform option
     std::string galaxy_platform_text = "Select platform\n";
-    for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i)
+    for (const auto& platform : GlobalConstants::PLATFORMS)
     {
-        galaxy_platform_text += GlobalConstants::PLATFORMS[i].str + " = " + GlobalConstants::PLATFORMS[i].regexp + "\n";
+        galaxy_platform_text += platform.str + " = " + platform.regexp + "\n";
     }
 
     // Create help text for --language option
     std::string language_text = "Select which language installers are downloaded\n";
-    for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i)
+    for (const auto& language : GlobalConstants::LANGUAGES)
     {
-        language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n";
+        language_text += language.str + " = " + language.regexp + "\n";
     }
     language_text += "All = all";
     language_text += "\n\n" + priority_help_text;
     language_text += "\nExample: German if available otherwise English and French: de,en+fr";
 
     // Create help text for --galaxy-language option
     std::string galaxy_language_text = "Select language\n";
-    for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i)
+    for (const auto& language : GlobalConstants::LANGUAGES)
     {
-        galaxy_language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n";
+        galaxy_language_text += language.str + " = " + language.regexp + "\n";
     }
 
     // Create help text for --galaxy-arch option
     std::string galaxy_arch_text = "Select architecture\n";
-    for (unsigned int i = 0; i < GlobalConstants::GALAXY_ARCHS.size(); ++i)
+    for (const auto& arch : GlobalConstants::GALAXY_ARCHS)
     {
-        galaxy_arch_text += GlobalConstants::GALAXY_ARCHS[i].str + " = " + GlobalConstants::GALAXY_ARCHS[i].regexp + "\n";
+        galaxy_arch_text += arch.str + " = " + arch.regexp + "\n";
     }
 
     // Create help text for --subdir-galaxy-install option
@@ -125,9 +128,9 @@ int main(int argc, char *argv[])
 
     // Create help text for --galaxy-cdn-priority option
     std::string galaxy_cdn_priority_text = "Set priority for used CDNs\n";
-    for (unsigned int i = 0; i < GlobalConstants::GALAXY_CDNS.size(); ++i)
+    for (const auto& cdn : GlobalConstants::GALAXY_CDNS)
     {
-        galaxy_cdn_priority_text += GlobalConstants::GALAXY_CDNS[i].str + " = " + GlobalConstants::GALAXY_CDNS[i].regexp + "\n";
+        galaxy_cdn_priority_text += cdn.str + " = " + cdn.regexp + "\n";
     }
     galaxy_cdn_priority_text += "\n" + priority_help_text;
 
@@ -140,22 +143,23 @@ int main(int argc, char *argv[])
 
     // Help text for include and exclude options
     std::string include_options_text;
-    for (unsigned int i = 0; i < GlobalConstants::INCLUDE_OPTIONS.size(); ++i)
+    for (const auto& include_option : GlobalConstants::INCLUDE_OPTIONS)
     {
-        include_options_text += GlobalConstants::INCLUDE_OPTIONS[i].str + " = " + GlobalConstants::INCLUDE_OPTIONS[i].regexp + "\n";
+        include_options_text += include_option.str + " = " + include_option.regexp + "\n";
     }
     include_options_text += "All = all\n";
     include_options_text += "Separate with \",\" to use multiple values";
 
     // Create help text for --list-format option
     std::string list_format_text = "List games/tags\n";
-    for (unsigned int i = 0; i < GlobalConstants::LIST_FORMAT.size(); ++i)
+    for (const auto& list_format: GlobalConstants::LIST_FORMAT)
     {
-        list_format_text += GlobalConstants::LIST_FORMAT[i].str + " = " + GlobalConstants::LIST_FORMAT[i].regexp + "\n";
+        list_format_text += list_format.str + " = " + list_format.regexp + "\n";
     }
 
     std::string galaxy_product_id_install;
     std::string galaxy_product_id_show_builds;
+    std::string galaxy_product_id_show_build_files;
     std::string galaxy_product_id_show_cloud_paths;
     std::string galaxy_product_id_show_local_cloud_paths;
     std::string galaxy_product_cloud_saves;
@@ -242,6 +246,7 @@ int main(int argc, char *argv[])
 #endif
             ("tag", bpo::value<std::string>(&tags)->default_value(""), "Filter using tags. Separate with \",\" to use multiple values")
             ("blacklist", bpo::value<std::string>(&Globals::globalConfig.sBlacklistFilePath)->default_value(sDefaultBlacklistFilePath), "Filepath to blacklist")
+            ("whitelist", bpo::value<std::string>(&Globals::globalConfig.sWhitelistFilePath)->default_value(sDefaultWhitelistFilePath), "Filepath to whitelist")
             ("ignorelist", bpo::value<std::string>(&Globals::globalConfig.sIgnorelistFilePath)->default_value(sDefaultIgnorelistFilePath), "Filepath to ignorelist")
         ;
         // Commandline options (config file)
@@ -303,6 +308,7 @@ int main(int argc, char *argv[])
         options_cli_experimental.add_options()
             ("galaxy-install", bpo::value<std::string>(&galaxy_product_id_install)->default_value(""), "Install game using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
             ("galaxy-show-builds", bpo::value<std::string>(&galaxy_product_id_show_builds)->default_value(""), "Show game builds using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
+            ("galaxy-show-build-files", bpo::value<std::string>(&galaxy_product_id_show_build_files)->default_value(""), "Show game build files using [product_id/build_index]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
             ("galaxy-download-cloud-saves", bpo::value<std::string>(&galaxy_product_cloud_saves)->default_value(""), "Download cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
             ("galaxy-upload-cloud-saves", bpo::value<std::string>(&galaxy_upload_product_cloud_saves)->default_value(""), "Upload cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
             ("galaxy-show-cloud-saves", bpo::value<std::string>(&galaxy_product_id_show_cloud_paths)->default_value(""), "Show game cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
@@ -408,7 +414,23 @@ int main(int argc, char *argv[])
                     std::getline(ifs, line);
                     lines.push_back(std::move(line));
                 }
-                Globals::globalConfig.blacklist.initialize(lines);
+                Globals::globalConfig.blacklist = Filelist(lines);
+            }
+        }
+
+        if (boost::filesystem::exists(Globals::globalConfig.sWhitelistFilePath)) {
+            std::ifstream ifs(Globals::globalConfig.sWhitelistFilePath.c_str());
+            if (!ifs) {
+                std::cerr << "Could not open whitelist file: " << Globals::globalConfig.sWhitelistFilePath << std::endl;
+                return 1;
+            } else {
+                std::string line;
+                std::vector<std::string> lines;
+                while (!ifs.eof()) {
+                    std::getline(ifs, line);
+                    lines.push_back(std::move(line));
+                }
+                Globals::globalConfig.whitelist = Filelist(lines);
             }
         }
 
@@ -429,7 +451,7 @@ int main(int argc, char *argv[])
                     std::getline(ifs, line);
                     lines.push_back(std::move(line));
                 }
-                Globals::globalConfig.ignorelist.initialize(lines);
+                Globals::globalConfig.ignorelist = Filelist(lines);
             }
         }
 
@@ -460,7 +482,7 @@ int main(int argc, char *argv[])
                         std::getline(ifs, line);
                         lines.push_back(std::move(line));
                     }
-                    Globals::globalConfig.gamehasdlc.initialize(lines);
+                    Globals::globalConfig.gamehasdlc = Filelist(lines);
                 }
             }
         }
@@ -832,6 +854,16 @@ int main(int argc, char *argv[])
         }
         downloader.galaxyShowBuilds(product_id, build_index);
     }
+    else if (!galaxy_product_id_show_build_files.empty())
+    {
+        int build_index = -1;
+        std::vector<std::string> tokens = Util::tokenize(galaxy_product_id_show_build_files, "/");
+        std::string product_id = tokens[0];
+        if (tokens.size() == 2) {
+            build_index = std::stoi(tokens[1]);
+        }
+        downloader.galaxyShowBuildFiles(product_id, build_index);
+    }
     else if (!galaxy_product_id_show_cloud_paths.empty())
     {
         int build_index = -1;

man/lgogdownloader.supplemental.groff

@@ -44,6 +44,14 @@ Rp the_witcher_2/extras/extras_pack_3_hu_pl_ru_tr_zh_\.zip
 .br
 Rp the_witcher_2/extras/extras_pack_2_fr_it_jp_\.zip
 
+[whitelist]
+.fi
+Allows user to specify individual files that should always downloaded.
+.sp 1
+This follows the same syntax as the blacklist.
+.sp 1
+This takes precedent over the blacklist. If an item is match on both lists, the whitelist will take priority.
+
 [files]
 .fi
 .TP
@@ -64,6 +72,14 @@ Allows user to specify individual files that should not be downloaded.
 .br
 It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader.
 
+
+.TP
+\fI$XDG_CONFIG_HOME/lgogdownloader/whitelist.txt\fP
+Allows user to specify individual files that should always be downloaded.  The file has 
+the same format and interpretation as a blacklist.
+.br
+It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader.
+
 .TP
 \fI$XDG_CONFIG_HOME/lgogdownloader/ignorelist.txt\fP
 Allows user to specify individual files that should not be mentioned
@@ -165,4 +181,3 @@ You're allowed to "stack" codes in the priority string if needed. If you set \fB
 
 [availability]
 The latest version of this distribution is available from \fIhttps://github.com/Sude-/lgogdownloader\fP
-