From 024724c7804d78b1e84e558f8ddaea061bceca86 Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Mon, 9 Dec 2024 21:01:46 -0600 Subject: [PATCH 1/4] Issue 534: remove demo data from baseline.sql. --- .pytest.ini | 3 +- devstart.py | 4 + lute/book/model.py | 3 + lute/db/demo.py | 75 ++++++-- lute/db/schema/baseline.sql | 129 +------------ lute/db/schema/empty.sql | 203 -------------------- lute/dev_api/routes.py | 1 + lute/language/routes.py | 6 +- lute/language/service.py | 96 +++++---- lute/main.py | 6 + lute/utils/debug_helpers.py | 8 +- tasks.py | 42 +--- tests/conftest.py | 4 +- tests/features/test_rendering.py | 4 + tests/features/test_term_import.py | 6 +- tests/unit/backup/test_backup.py | 10 + tests/unit/book/test_Repository.py | 6 +- tests/unit/book/test_datatables.py | 6 +- tests/unit/cli/test_language_term_export.py | 16 +- tests/unit/db/test_demo.py | 92 +++++++-- tests/unit/db/test_management.py | 3 - tests/unit/language/test_service.py | 37 +++- tests/unit/models/test_Language.py | 8 + tests/unit/utils/test_formutils.py | 10 +- 24 files changed, 312 insertions(+), 466 deletions(-) delete mode 100644 lute/db/schema/empty.sql diff --git a/.pytest.ini b/.pytest.ini index 45ce83228..0e368e257 100644 --- a/.pytest.ini +++ b/.pytest.ini @@ -29,5 +29,4 @@ markers = # Rather than sorting out how to add a flask cli command # that has access to the configured app and context, # I'm just using some markers to reset/wipe the dev db. - dbdemoload: cli hack to load the dev db with demo data - dbwipe: cli hack to wipe the dev db + dbreset: cli hack to wipe the dev db and set the LoadDemoData flag diff --git a/devstart.py b/devstart.py index 7c58574b7..cf28ceb00 100644 --- a/devstart.py +++ b/devstart.py @@ -24,6 +24,8 @@ from lute import __version__ from lute.app_factory import create_app from lute.config.app_config import AppConfig +from lute.db import db +from lute.db.demo import load_demo_data log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) @@ -46,6 +48,8 @@ def dev_print(s): config_file = AppConfig.default_config_filename() dev_print("") app = create_app(config_file, output_func=dev_print) + with app.app_context(): + load_demo_data(db.session) ac = AppConfig(config_file) dev_print(f"\nversion {__version__}") diff --git a/lute/book/model.py b/lute/book/model.py index 872a948ef..9224fd247 100644 --- a/lute/book/model.py +++ b/lute/book/model.py @@ -101,6 +101,9 @@ def _build_db_book(self, book): lang = lang_repo.find(book.language_id) elif book.language_name: lang = lang_repo.find_by_name(book.language_name) + if lang is None: + msg = f"No language matching id={book.language_id} or name={book.language_name}" + raise RuntimeError(msg) b = None if book.id is None: diff --git a/lute/db/demo.py b/lute/db/demo.py index 98f7e5709..0608a0bad 100644 --- a/lute/db/demo.py +++ b/lute/db/demo.py @@ -12,7 +12,7 @@ from lute.language.service import Service from lute.book.model import Repository from lute.book.stats import Service as StatsService -from lute.models.repositories import SystemSettingRepository +from lute.models.repositories import SystemSettingRepository, LanguageRepository import lute.db.management @@ -38,15 +38,32 @@ def _demo_languages(): ] -def contains_demo_data(session): - """ - True if IsDemoData setting is present. - """ +def set_load_demo_flag(session): + "Set the flag." + repo = SystemSettingRepository(session) + repo.set_value("LoadDemoData", True) + session.commit() + + +def remove_load_demo_flag(session): + "Set the flag." repo = SystemSettingRepository(session) - ss = repo.get_value("IsDemoData") - if ss is None: - return False - return True + repo.delete_key("LoadDemoData") + session.commit() + + +def _flag_exists(session, flagname): + "True if flag exists, else false." + repo = SystemSettingRepository(session) + return repo.key_exists(flagname) + + +def should_load_demo_data(session): + return _flag_exists(session, "LoadDemoData") + + +def contains_demo_data(session): + return _flag_exists(session, "IsDemoData") def remove_flag(session): @@ -92,13 +109,13 @@ def delete_demo_data(session): def load_demo_languages(session): """ - Load selected predefined languages. Assume everything is supported. + Load selected predefined languages, if they're supported. This method will also be called during acceptance tests, so it's public. """ demo_langs = _demo_languages() service = Service(session) - langs = [service.get_language_def(langname)["language"] for langname in demo_langs] + langs = [service.get_language_def(langname).language for langname in demo_langs] supported = [lang for lang in langs if lang.is_supported] for lang in supported: session.add(lang) @@ -106,15 +123,22 @@ def load_demo_languages(session): def load_demo_stories(session): - "Load the stories." + "Load the stories for any languages already loaded." demo_langs = _demo_languages() service = Service(session) langdefs = [service.get_language_def(langname) for langname in demo_langs] - langdefs = [d for d in langdefs if d["language"].is_supported] + + langrepo = LanguageRepository(session) + langdefs = [ + d + for d in langdefs + if d.language.is_supported + and langrepo.find_by_name(d.language.name) is not None + ] r = Repository(session) for d in langdefs: - for b in d["books"]: + for b in d.books: r.add(b) r.commit() @@ -126,12 +150,33 @@ def load_demo_stories(session): svc.refresh_stats() +def _db_has_data(session): + "True of the db contains any language data." + sql = "select LgID from languages limit 1" + r = session.execute(text(sql)).first() + return r is not None + + def load_demo_data(session): """ Load the data. """ + if _db_has_data(session): + remove_load_demo_flag(session) + return + + repo = SystemSettingRepository(session) + do_load = repo.get_value("LoadDemoData") + if do_load is None: + # Only load if flag is explicitly set. + return + + do_load = bool(int(do_load)) + if not do_load: + return + load_demo_languages(session) load_demo_stories(session) - repo = SystemSettingRepository(session) + remove_load_demo_flag(session) repo.set_value("IsDemoData", True) session.commit() diff --git a/lute/db/schema/baseline.sql b/lute/db/schema/baseline.sql index 46ae226f8..40ea7832e 100644 --- a/lute/db/schema/baseline.sql +++ b/lute/db/schema/baseline.sql @@ -1,5 +1,5 @@ -- ------------------------------------------ --- Baseline db with demo data. +-- Baseline db with flag to load demo data. -- Migrations tracked in _migrations, settings reset. -- Generated from 'inv db.export.baseline' -- ------------------------------------------ @@ -138,37 +138,13 @@ CREATE TABLE IF NOT EXISTS "texts" ( PRIMARY KEY ("TxID"), FOREIGN KEY("TxBkID") REFERENCES "books" ("BkID") ON UPDATE NO ACTION ON DELETE CASCADE ); -INSERT INTO texts VALUES(1,1,1,replace('مرحبا، كيف حالك ؟\nمرحبا, أنا بخير\nهل انت جديدٌ هنا؟ لم أراك من قبل\nانا طالب جديد.لقد وصلت البارحة\nانا محمد, تشرفت بلقائك\n\nشجرة الحياة\n\nتحكي هذه القصة عن ولد صغير يُدعى «يوسف»، يعيش مع أمه الأرملة الفقيرة، يساعدها ويحنو عليها ويحبها حبًا جمًا. وفي يوم من الأيام يصيب المرض أم يوسف ويشتد عليها، ولا يعرف يوسف ماذا يفعل لإنقاذها، فلا يجد أمامه سوى اللجوء إلى الجِنِّيَّة «وِداد» التي تدله على شجرة فيها الشفاء لأمه، هذه الشجرة تقع في أعلى الجبل المقابل لمنزلهم، وعلى يوسف أن يتسلق هذا الجبل ويواجه المخاطر من أجل أن يأتي لأمه بالدواء الموجود في أوراق هذه الشجرة، فهل سينجح يوسف في ذلك؟ وماذا ينتظره من مخاطر وأهوال؟','\n',char(10)),NULL,115); -INSERT INTO texts VALUES(2,2,1,'北冥有魚，其名為鯤。鯤之大，不知其幾千里也。化而為鳥，其名為鵬。鵬之背，不知其幾千里也；怒而飛，其翼若垂天之雲。是鳥也，海運則將徙於南冥。南冥者，天池也。齊諧者，志怪者也。諧之言曰：「鵬之徙於南冥也，水擊三千里，摶扶搖而上者九萬里，去以六月息者也。」野馬也，塵埃也，生物之以息相吹也。天之蒼蒼，其正色邪？其遠而無所至極邪？其視下也亦若是，則已矣。且夫水之積也不厚，則負大舟也無力。覆杯水於坳堂之上，則芥為之舟，置杯焉則膠，水淺而舟大也。風之積也不厚，則其負大翼也無力。故九萬里則風斯在下矣，而後乃今培風；背負青天而莫之夭閼者，而後乃今將圖南。',NULL,228); -INSERT INTO texts VALUES(3,2,2,'蜩與學鳩笑之曰：「我決起而飛，槍1榆、枋，時則不至而控於地而已矣，奚以之九萬里而南為？」適莽蒼者三湌而反，腹猶果然；適百里者宿舂糧；適千里者三月聚糧。之二蟲又何知！小知不及大知，小年不及大年。奚以知其然也？朝菌不知晦朔，蟪蛄不知春秋，此小年也。楚之南有冥靈者，以五百歲為春，五百歲為秋；上古有大椿者，以八千歲為春，八千歲為秋。而彭祖乃今以久特聞，眾人匹之，不亦悲乎！',NULL,154); -INSERT INTO texts VALUES(4,3,1,'V jedné rozpadlé chaloupce žil tatínek, maminka a jejich chlapec Tonda. Byli velmi chudí, do střechy jim teklo a často neměli ani na jídlo. Tonda, aby rodičům pomohl, pracoval jako pasáček ovcí. Ale i tak neměl ani na pořádné oblečení, chodil v roztrhaných kalhotách, do kabátku mu táhlo a jeho čepice vypadala stejně otrhaně jako zbytek oděvu. I přesto si každé ráno cestou na pastvu vesele pohvizdoval.',NULL,67); -INSERT INTO texts VALUES(5,4,1,replace('Welcome to Lute! This short guide should get you going.\n\nNavigation\nThis tutorial has multiple pages. At the top of the page is a slider to navigate forwards and backwards, or you can click the arrows at either end of the slider.\n\n1. The Basics\nAll of these words are blue because they are "unknown" - according to Lute, this is the first time you''re seeing these words.\nYou can click on a word, and create a definition. For example, click on this word: elephant.\nWhen the form pops up in the right-hand frame, a dictionary is loaded below. Copy-paste something from the dictionary into the translation, or make up your own, mark the status, add some tags if you want (eg, type "noun" in the tags field), and click save. From now on, every English text that you read that contains the word "elephant" will show the status. If you hover over any "elephant", you''ll see some information.\n\n1.1 Multiple dictionaries.\nLanguages can have multiple dictionaries, configured in the "Languages" link on the homepage. Each dictionary is shown as a small tab. For this demo, English has been configured with two dictionaries. The second dictionary will open a popup window. If you have many dictionaries, the extra ones will be shown in a small list box next to the tabs.\n\n1.2 Images\nLute can do a simple image search for terms. Next to the Sentences tab is a small image button.','\n',char(10)),NULL,242); -INSERT INTO texts VALUES(6,4,2,replace('If you click on it, you''ll see some happy elephants (if you clicked on elephant!). If you click on one of the images shown in the list, that image is saved in your data/userimages folder. When you hover over the word in the reading pane, that picture is included in the word hover. Try adding an image for your elephant by clicking on the term, clicking the image icon, and clicking a picture you like. Then hover over your elephant.\nNote: sometimes these images make _no sense_ -- it''s using Bing image search, and it does the best it can with the limited context it has.\nTo delete an image added to your term, click on it. Its border will change to red. Hit Delete or Backspace to delete it, and then save the term (you must save!).\n\n2. Multi-word Terms\nYou can create multi-word terms by clicking and dragging across multiple words, then release the mouse. Try creating a term for the phrase "the cat''s pyjamas", and add a translation and set the status.\n(A brief side note: Lute keeps track of where you are in any text. If you click the Home link above to leave this tutorial, and later click the Tutorial link from the Text listing, Lute will open the last page you were at.)\n\n3. Parent Terms\nSometimes it helps to associate terms with a "parent".','\n',char(10)),NULL,234); -INSERT INTO texts VALUES(7,4,3,replace('For example, the verb "to have" is conjugated in various forms as "I have a cold", "he has a dog", "they had dinner". First create a Term for "have". Then create a Term for "has", and in the Parent field start typing "have". Lute will show the existing Term "have" in the drop down, and if you select it, "has" will be associated with that on save, and when you hover over "has" you''ll see the parent''s information as well.\nIf you enter a non-existent Parent word, Lute will create a placeholder Term for that Parent, copying some content from your term. For example, try creating a Term for the word "dogs", associating it with the non-existent Term "dog". When you save "dogs", both will be updated.\nTerms can have multiple parents, too. Hit the Enter (or Return) key after each parent. For example, if you wanted to associate the Term "puppies" with both "puppy" and "dog", click on "puppies", and in the Parents text box type "puppy", hit Enter, type "dog", and hit Enter. Sometimes this is necessary: for example, in Spanish, "se sienta" can either be a conjugation of "sentirse" (to feel) or "sentarse" (to sit), depending on the context.\n\n4. Mark the remainder as "Well Known"\nWhen you''re done creating Terms on a page, you will likely still have a bunch of blue words, or "unknowns", left over, even though you really know these words.','\n',char(10)),NULL,242); -INSERT INTO texts VALUES(8,4,4,replace('You can set all of these to "Well Known" and move to the next page in one shot with the green checkmark at the bottom of the page. Try that now to see what happens, and then come back to this page using the arrows in the header to finish reading this page.\nThe ">" link moves to the next page as well, without setting the unknown terms to "Well Known." This can be useful if you''re reading quickly, without stopping to define every last term in detail.\nNote: both of these links also mark the page as "Read", which Lute uses when it searches for references to terms you create. There''s more on this in the tutorial follow-up, which you should read after this tutorial.\n\n5. Keyboard shortcuts\nThe "hamburger menu" next to the Lute logo at the top left of the reading screen opens up a small menu of items, including a link to some keyboard shortcuts. This section introduces a few of them.\n\n5.1 Updating Status\nIf you''ve worked through the tutorial, you''ll have noted that words are underlined in blue when you move the mouse over them. You can quickly change the status of the current word by hitting 1, 2, 3, 4, 5, w (for Well-Known), or i (for Ignore). Try hovering over the following words and hit the status buttons: apple, banana, cranberry, donut.\nIf you click on a word, it''s underlined in red, and the Term edit form is shown.','\n',char(10)),NULL,246); -INSERT INTO texts VALUES(9,4,5,replace('Before you switch over to the Term editing form, you can still update its status using the hotkeys above, or by using the up and down arrows. You can jump over to the edit form by hitting Tab, and then start editing.\nWhen a word has been clicked, it''s "active", so it keeps the focus. Hovering the mouse over other words won''t underline them in blue anymore, and hitting status update hotkeys (1 - 5, w, i) will only update the active word. To "un-click" a word underlined in red, click it again, or hit Escape or Return. Then you''ll be back in "Hover mode". In "Hover mode", the hotkeys 1-5, w, and i still update the status, but the arrow keys just scroll the window. Try clicking and un-clicking or Escaping any word in this paragraph to get a feel for it.\nNote that for the keyboard shortcuts to work, the reading pane (where the text is) must have the "focus". Click anywhere on the reading pane to re-establish focus.\n\n5.1 Bulk updates\nIf you hold down Shift and click a bunch of words, you can bulk update their statuses. This works for the up and down arrow keys as well.\n\n5.2 Arrow keys\nThe Right and Left arrow keys click the next and previous words. Hit Escape or Return to get back to "hover mode".\n\n5.3 Copying text\nWhen a word is hovered over or clicked, hit "c" to copy that word''s sentence to your clipboard.','\n',char(10)),NULL,248); -INSERT INTO texts VALUES(10,4,6,replace('Hit "C" to copy the word''s full paragraph (multiple sentences). You can also copy arbitrary sections of text by holding down the Shift key while highlighting the text with your mouse.\n\n6. Next steps\nAll done this text!\nLute keeps track of all of this in your database, so any time you create or import a new Book, all the info you''ve created is carried forward.\nThere''s a tutorial follow-up: go to the Home screen, and click the "Tutorial follow-up" in the table.','\n',char(10)),NULL,87); -INSERT INTO texts VALUES(11,5,1,replace('Hopefully you''ve gone through the Tutorial, and created some Terms.\nFrom the Tutorial, you''ve already told Lute that you know most of the words on this page. You can hover over words to see information about them, such as your information you might have added about dogs.\nThere are still a few blue words, which according to Lute are still "unknown" to you. You can process them like you did on the last text.\n(fyi - If a text has a spelling mikstaske, you can edit it by clicking the "hamburger menu" -- three lines -- in the top left corner of this screen, and click "Edit page". If you''d like, correct the mistake now, and resave this text.)\nNow we''ll do a brief spin through a few other things Lute does. You can read about them and other features in the manual too.\n\n1. The Menus\nIn case you missed it, on the Home screen there are some menu bar items on the top right. Go back there and hover over them to see what you can do. This is all demo data, so you can do what you want. (But don''t delete the tutorials until you''ve gone through them.)\n\n2. Term Sentences\nIn the "Term" edit form, you can click on the "Sentences" tab to see where that term or its relations have been used. Click on "elephant", and then click the Sentences tab to see where that term has been used.','\n',char(10)),NULL,245); -INSERT INTO texts VALUES(12,5,2,replace('You''re only shown sentences on pages that have been marked "Read", using the controls in the footer of the reading screen, i.e. the green checkmark or the ">". This ensures that you only see references that you have already seen before, so you don''t get overwhelmed with new vocabulary, and avoids spoilers of the material you''re reading.\n\n3. Archiving, Unarchiving, and Deleting Books\nWhen you''re done reading a book, you can Archive or Delete it.\nArchiving just removes the book from your book listing on the home screen, and you can unarchive at any time. The sentences for archived books are still available for searching with the Term "Sentences" link.\nOn the last page of every book, Lute shows a link for you to archive the book. You can also archive it from the Home screen by clicking on the "Archive" action (the image with the little down arrow) in the right-most column. To unarchive the text, go to Home, Book Archive, and click the "Unarchive" action (the little up arrow).\nDeleting a book completely removes it and its sentences.\nNeither archiving nor deleting touch any Terms you''ve created.\n\n4. Audio\nYou can add an audio file (mp3, wav, or ogg) to your books, so you can read along to an audio track. See the Lute manual for more notes on usage and tips.\n\n5. Themes and toggling highlighting\nLute has a few themes to make reading more pleasant.','\n',char(10)),NULL,242); -INSERT INTO texts VALUES(13,5,3,replace('From the "hamburger menu" on the reading screen you can switch to the next theme, or hit the hotkey (m). You can also toggle highlights, because sometimes they get distracting.\n\n===\n\nThose are the the core feature of Lute! There are some sample stories for other languages. Try those out or create your own.\nWhen you''re done with the demo, go back to the Home screen and click the link to clear out the database. Lute will delete all of the demo data, and you can get started. You''ll be prompted to create your first language, and then you can create your first book. Lute will then ask you to specify your backup preferences, and with that all done, you''ll be off and running.\nThere is a Lute Discord and manual as well -- see the "About" menu bar.\nI hope that you find Lute a fun tool to use for learning languages. Cheers and best wishes!','\n',char(10)),NULL,158); -INSERT INTO texts VALUES(14,6,1,replace('Il était une fois trois ours: un papa ours, une maman ours et un bébé ours. Ils habitaient tous ensemble dans une maison jaune au milieu d''une grande forêt.\n\nUn jour, Maman Ours prépara une grande marmite de porridge délicieux et fumant pour le petit déjeuner. Il était trop chaud pour pouvoir être mangé, alors les ours décidèrent d''aller se promener en attendant que le porridge refroidisse.','\n',char(10)),NULL,69); -INSERT INTO texts VALUES(15,7,1,replace('Es hatte ein Mann einen Esel, der schon lange Jahre die Säcke unverdrossen zur Mühle getragen hatte, dessen Kräfte aber nun zu Ende gingen, sodass er zur Arbeit immer untauglicher wurde. Da dachte der Herr daran, ihn aus dem Futter zu schaffen, aber der Esel merkte, dass kein guter Wind wehte, lief fort und machte sich auf den Weg nach Bremen; dort, meinte er, könnte er ja Stadtmusikant werden.\n\nAls er ein Weilchen fortgegangen war, fand er einen Jagdhund auf dem Weg liegen, der japste wie einer, der sich müde gelaufen hat. "Nun, was japst du so, Packan?" fragte der Esel. "Ach," sagte der Hund, "weil ich alt bin und jeden Tag schwächer werde, auch auf der Jagd nicht mehr fort kann, hat mich mein Herr wollen totschlagen, da hab ich Reißaus genommen; aber womit soll ich nun mein Brot verdienen?" - "Weißt du was?" sprach der Esel, "ich gehe nach Bremen und werde dort Stadtmusikant, geh mit und lass dich auch bei der Musik annehmen. Ich spiele die Laute, und du schlägst die Pauken.','\n',char(10)),NULL,174); -INSERT INTO texts VALUES(16,8,1,replace('Πέτρος: Γεια σου, Νίκη. Ο Πέτρος είμαι.\nΝίκη: Α, γεια σου Πέτρο. Τι κάνεις;\nΠέτρος: Μια χαρά. Σε παίρνω για να πάμε καμιά βόλτα αργότερα. Τι λες;\nΝίκη: Α, ωραία. Κι εγώ θέλω να βγω λίγο. Συνέχεια διαβάζω για τις εξετάσεις… κουράστηκα πια. Πού λες να πάμε;\nΠέτρος: Στη γνωστή καφετέρια στην πλατεία. Θα είναι και άλλα παιδιά από την τάξη μας εκεί.\nΝίκη: Ναι; Ποιοι θα είναι;\nΠέτρος: Ο Γιάννης, ο Αντρέας και η Ελπίδα.\nΝίκη: Ωραία. Θα πάτε και πουθενά αλλού μετά;\nΠέτρος: Ναι, λέμε να πάμε στον κινηματογράφο που είναι κοντά στην καφετέρια. Παίζει μια κωμωδία.\nΝίκη: Α, δεν μπορώ να καθίσω έξω μέχρι τόσο αργά. Πρέπει να γυρίσω σπίτι για να διαβάσω.\nΠέτρος: Έλα τώρα. Διαβάζεις αύριο…\nΝίκη: Όχι, όχι, αδύνατον. Είμαι πολύ πίσω στο διάβασμά μου.\nΠέτρος: Καλά, έλα μόνο στην καφετέρια τότε. Θα περάσω να σε πάρω γύρω στις έξι να πάμε μαζί. Εντάξει;\nΝίκη: Εντάξει. Γεια.\nΠέτρο: Τα λέμε. Γεια.','\n',char(10)),NULL,157); -INSERT INTO texts VALUES(17,9,1,'अनुच्छेद १(एक): सभी मनुष्य जन्म से स्वतन्त्र तथा मर्यादा और अधिकारों में समान होते हैं। वे तर्क और विवेक से सम्पन्न हैं तथा उन्हें भ्रातृत्व की भावना से परस्पर के प्रति कार्य करना चाहिए।',NULL,35); -INSERT INTO texts VALUES(18,10,1,replace('北風と太陽\n\n「おれの方が強い。」「いいや、ぼくの方が強い。」\n北風と太陽の声が聞こえます。二人はどちらの力が強いかでケンカをしているようです。\n「太陽が毎日元気だから、暑くてみんな困っているよ。おれが涼しい風を吹くと、みんな嬉しそうだ。」','\n',char(10)),NULL,64); -INSERT INTO texts VALUES(19,11,1,'Встреча с медведем может быть очень опасна. Русские люди любят ходить в лес и собирать грибы и ягоды. Они делают это с осторожностью, так как медведи тоже очень любят ягоды и могут напасть на человека. Медведь ест все: ягоды, рыбу, мясо и даже насекомых. Особенно он любит мед.',NULL,48); -INSERT INTO texts VALUES(20,12,1,replace('काशीनगरे एकः पण्डितः अस्ति । पण्डितसमीपम् एकः शिष्यः आगच्छति । शिष्यः वदति - "आचार्य!\nविद्याभ्यासार्थम् आगतः ।" पण्डितः शिष्यबुद्धिपरीक्षार्थं पृच्छति - "वत्स, देवः कुत्र अस्ति?" शिष्यः वदति -\n''गुरो! देवः कुत्र नास्ति? कृपया भवान् एव समाधानं वदतु ।" सन्तुष्टः गुरुः वदति - "दैवः सर्वत्र अस्ति । देवः\nसर्वव्यापी । त्वं बुद्धिमान ।अतः विद्याभ्यासार्थम् अत्रैव वस।"','\n',char(10)),NULL,45); -INSERT INTO texts VALUES(21,13,1,replace('dhṛtarāṣṭro rājā.\nkiṃ dhṛtarāṣṭro mantrī?\ndhṛtarāṣṭro na mantrī.\ndhṛtarāṣṭro rājā.\n\nsaṃjayaḥ kaḥ?\nkiṃ saṃjayo rājā?\nsaṃjayo na rājā.\nsaṃjayo mantrī.\nsaṃjayo dhṛtarāṣṭrasya mantrī.\n\nsaṃjayo dhṛtarāṣṭraṃ gacchati.\nsaṃjayo rājānaṃ dhṛtarāṣṭraṃ gacchati.\ndhṛtarāṣṭraḥ:\n saṃjaya!\n duryodhanaḥ kiṃ karoti?\n\nduryodhanaḥ kaḥ?\nkiṃ duryodhano mantrī?\nduryodhano na mantrī.\nduryodhano dhṛtarāṣṭrasya putraḥ.\nduryodhano rāja­putraḥ.','\n',char(10)),NULL,49); -INSERT INTO texts VALUES(22,14,1,replace('धृतराष्ट्रो राजा।\nकिं धृतराष्ट्रो मन्त्री?\nधृतराष्ट्रो न मन्त्री।\nधृतराष्ट्रो राजा।\n\nसंजयः कः?\nकिं संजयो राजा?\nसंजयो न राजा।\nसंजयो मन्त्री।\nसंजयो धृतराष्ट्रस्य मन्त्री।\n\nसंजयो धृतराष्ट्रं गच्छति।\nसंजयो राजानं धृतराष्ट्रं गच्छति।\nधृतराष्ट्रः —\n संजय !\n दुर्योधनः किं करोति?\n\nदुर्योधनः कः?\nकिं दुर्योधनो मन्त्री?\nदुर्योधनो न मन्त्री।\nदुर्योधनो धृतराष्ट्रस्य पुत्रः।\nदुर्योधनो राज­पुत्रः।','\n',char(10)),NULL,49); -INSERT INTO texts VALUES(23,15,1,replace('Érase una vez un muchacho llamado Aladino que vivía en el lejano Oriente con su madre, en una casa sencilla y humilde. Tenían lo justo para vivir, así que cada día, Aladino recorría el centro de la ciudad en busca de algún alimento que llevarse a la boca.\n\nEn una ocasión paseaba entre los puestos de fruta del mercado, cuando se cruzó con un hombre muy extraño con pinta de extranjero. Aladino se quedó sorprendido al escuchar que le llamaba por su nombre.','\n',char(10)),NULL,83); -INSERT INTO texts VALUES(24,16,1,replace('Büyük ağaç eskiden aşılanmış ve her yıl güzel, iri, pembe şeftaliler verirmiş, insanın eline sığmazmış bu şeftaliler. Öyle güzelmişler ki insan yemeye kıyamazmış onları. Bahçıvan, bu büyük ağacı yabancı bir uzmanın kendi ülkesinden getirdiği bir tohumla aşıladığını söylermiş. Belli ki böyle masraf edilen bir ağaçta yetişen şeftaliler oldukça değerliymiş.\n\nİki ağacın da gövdelerine nazar değmesin diye birer nazarlık asılıymış.\n\nAğaçlardan küçük olanında her yıl bin tane çiçek açarmış ama bir tek şeftali bile yetişmezmiş üzerinde. Ya çiçekleri dökülürmüş, ya da ham şeftaliler kuruyup dallardan düşermiş. Bahçıvan küçük ağaç için elinden geleni yapmış ama değişen bir şey olmamış. Yıllar geçtikçe dalları ve yaprakları çoğalmış ama bir tek şeftali bile görünmemiş üzerinde.','\n',char(10)),NULL,110); CREATE TABLE IF NOT EXISTS "settings" ( "StKey" VARCHAR(40) NOT NULL, "StKeyType" TEXT NOT NULL, "StValue" TEXT NULL, PRIMARY KEY ("StKey") ); -INSERT INTO settings VALUES('IsDemoData','system','1'); +INSERT INTO settings VALUES('LoadDemoData','system','1'); CREATE TABLE IF NOT EXISTS "books" ( "BkID" INTEGER NOT NULL , "BkLgID" INTEGER NOT NULL , @@ -182,22 +158,6 @@ CREATE TABLE IF NOT EXISTS "books" ( PRIMARY KEY ("BkID"), FOREIGN KEY("BkLgID") REFERENCES "languages" ("LgID") ON UPDATE NO ACTION ON DELETE CASCADE ); -INSERT INTO books VALUES(1,1,'Examples',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(2,2,'逍遙遊',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(3,3,'Hrad Cimburk – Jak vzal vítr pasáčkovi čepici',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(4,4,'Tutorial',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(5,4,'Tutorial follow-up',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(6,5,'Boucles d’or et les trois ours',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(7,6,'Die Bremer Stadtmusikanten',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(8,7,'Γεια σου, Νίκη. Ο Πέτρος είμαι.',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(9,8,'Universal Declaration of Human Rights',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(10,9,'北風と太陽 - きたかぜたいよう',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(11,10,'медведь',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(12,11,'बुद्धिमान् शिष्यः',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(13,11,'Bhagavad Ghita (Latin)',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(14,11,'Bhagavad Ghita (Devanagari)',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(15,12,'Aladino y la lámpara maravillosa',NULL,0,0,NULL,NULL,NULL); -INSERT INTO books VALUES(16,13,'Büyük ağaç',NULL,0,0,NULL,NULL,NULL); CREATE TABLE IF NOT EXISTS "bookstats" ( "BkID" INTEGER NOT NULL , "distinctterms" INTEGER NULL , @@ -207,22 +167,6 @@ CREATE TABLE IF NOT EXISTS "bookstats" ( PRIMARY KEY ("BkID"), FOREIGN KEY("BkID") REFERENCES "books" ("BkID") ON UPDATE NO ACTION ON DELETE CASCADE ); -INSERT INTO bookstats VALUES(1,100,100,100,'{"0": 100, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(2,170,170,100,'{"0": 170, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(3,57,57,100,'{"0": 57, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(4,361,361,100,'{"0": 361, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(5,247,247,100,'{"0": 247, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(6,49,49,100,'{"0": 49, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(7,120,120,100,'{"0": 120, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(8,99,99,100,'{"0": 99, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(9,30,30,100,'{"0": 30, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(10,41,41,100,'{"0": 41, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(11,40,40,100,'{"0": 40, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(12,33,33,100,'{"0": 33, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(13,19,19,100,'{"0": 19, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(14,19,19,100,'{"0": 19, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(15,63,63,100,'{"0": 63, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); -INSERT INTO bookstats VALUES(16,85,85,100,'{"0": 85, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "98": 0, "99": 0}'); CREATE TABLE languagedicts ( "LdID" INTEGER NOT NULL, "LdLgID" INTEGER NOT NULL, @@ -234,62 +178,6 @@ CREATE TABLE languagedicts ( PRIMARY KEY ("LdID"), FOREIGN KEY("LdLgID") REFERENCES "languages" ("LgID") ON UPDATE NO ACTION ON DELETE CASCADE ); -INSERT INTO languagedicts VALUES(1,1,'terms','embeddedhtml','https://www.arabicstudentsdictionary.com/search?q=[LUTE]',1,1); -INSERT INTO languagedicts VALUES(2,1,'terms','popuphtml','https://translate.google.com/?hl=en&sl=ar&tl=en&text=[LUTE]&op=translate',1,2); -INSERT INTO languagedicts VALUES(3,1,'sentences','popuphtml','https://translate.google.com/?hl=en&sl=ar&tl=en&text=[LUTE]',1,3); -INSERT INTO languagedicts VALUES(4,2,'terms','embeddedhtml','https://www.archchinese.com/chinese_english_dictionary.html?find=[LUTE]',1,1); -INSERT INTO languagedicts VALUES(5,2,'sentences','popuphtml','https://www.deepl.com/translator#ch/en/[LUTE]',1,2); -INSERT INTO languagedicts VALUES(6,3,'terms','embeddedhtml','https://slovniky.lingea.cz/Anglicko-cesky/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(7,3,'terms','popuphtml','https://slovnik.seznam.cz/preklad/cesky_anglicky/[LUTE]',1,2); -INSERT INTO languagedicts VALUES(8,3,'sentences','popuphtml','https://www.deepl.com/translator#cs/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(9,4,'terms','embeddedhtml','https://simple.wiktionary.org/wiki/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(10,4,'terms','popuphtml','https://www.collinsdictionary.com/dictionary/english/[LUTE]',1,2); -INSERT INTO languagedicts VALUES(11,4,'sentences','popuphtml','https://www.deepl.com/translator#en/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(12,4,'terms','popuphtml','https://conjugator.reverso.net/conjugation-english-verb-[LUTE].html',1,4); -INSERT INTO languagedicts VALUES(13,5,'terms','embeddedhtml','https://www.wordreference.com/fren/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(14,5,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#French',1,2); -INSERT INTO languagedicts VALUES(15,5,'sentences','popuphtml','https://www.deepl.com/translator#fr/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(16,5,'terms','popuphtml','https://www.larousse.fr/dictionnaires/francais/[LUTE]',1,4); -INSERT INTO languagedicts VALUES(17,5,'terms','popuphtml','https://conjugator.reverso.net/conjugation-french-verb-[LUTE].html',1,5); -INSERT INTO languagedicts VALUES(18,6,'terms','embeddedhtml','https://www.dict.cc/?s=[LUTE]',1,1); -INSERT INTO languagedicts VALUES(19,6,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#German',1,2); -INSERT INTO languagedicts VALUES(20,6,'sentences','popuphtml','https://www.deepl.com/translator#de/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(21,6,'terms','popuphtml','https://www.duden.de/suchen/dudenonline/[LUTE]',1,4); -INSERT INTO languagedicts VALUES(22,6,'terms','popuphtml','https://conjugator.reverso.net/conjugation-german-verb-[LUTE].html',1,5); -INSERT INTO languagedicts VALUES(23,7,'terms','embeddedhtml','https://www.wordreference.com/gren/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(24,7,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Greek',1,2); -INSERT INTO languagedicts VALUES(25,7,'sentences','popuphtml','https://www.deepl.com/translator#el/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(26,7,'terms','embeddedhtml','https://www.greek-language.gr/greekLang/modern_greek/tools/lexica/search.html?sin=all&lq=[LUTE]',1,4); -INSERT INTO languagedicts VALUES(27,7,'terms','embeddedhtml','https://cooljugator.com/gr/[LUTE]',1,5); -INSERT INTO languagedicts VALUES(28,8,'terms','embeddedhtml','https://www.boltidictionary.com/en/search?s=[LUTE]',1,1); -INSERT INTO languagedicts VALUES(29,8,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Hindi',1,2); -INSERT INTO languagedicts VALUES(30,8,'sentences','popuphtml','https://translate.google.com/?sl=hi&tl=en&text=[LUTE]',1,3); -INSERT INTO languagedicts VALUES(31,8,'terms','embeddedhtml','https://verbix.com/webverbix/go.php?&D1=47&T1=[LUTE]',1,4); -INSERT INTO languagedicts VALUES(32,9,'terms','embeddedhtml','https://jisho.org/search/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(33,9,'terms','popuphtml','https://www.japandict.com/?s=[LUTE]&lang=eng',1,2); -INSERT INTO languagedicts VALUES(34,9,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Japanese',1,3); -INSERT INTO languagedicts VALUES(35,9,'sentences','popuphtml','https://www.deepl.com/translator#jp/en/[LUTE]',1,4); -INSERT INTO languagedicts VALUES(36,9,'terms','embeddedhtml','https://www.weblio.jp/content/[LUTE]',1,5); -INSERT INTO languagedicts VALUES(37,9,'terms','popuphtml','https://conjugator.reverso.net/conjugation-japanese-verb-[LUTE].html',1,6); -INSERT INTO languagedicts VALUES(38,10,'terms','embeddedhtml','https://en.openrussian.org/?search=[LUTE]',1,1); -INSERT INTO languagedicts VALUES(39,10,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Russian',1,2); -INSERT INTO languagedicts VALUES(40,10,'sentences','popuphtml','https://www.deepl.com/translator#ru/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(41,10,'terms','embeddedhtml','https://gramota.ru/poisk?query=[LUTE]&mode=all',1,4); -INSERT INTO languagedicts VALUES(42,10,'terms','popuphtml','https://conjugator.reverso.net/conjugation-russian-verb-[LUTE].html',1,5); -INSERT INTO languagedicts VALUES(43,11,'terms','embeddedhtml','https://www.learnsanskrit.cc/translate?search=[LUTE]&dir=se',1,1); -INSERT INTO languagedicts VALUES(44,11,'terms','embeddedhtml','https://dsal.uchicago.edu/cgi-bin/app/sanskrit_query.py?qs=[LUTE]&searchhws=yes&matchtype=default',1,2); -INSERT INTO languagedicts VALUES(45,11,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Sanskrit',1,3); -INSERT INTO languagedicts VALUES(46,11,'sentences','popuphtml','https://translate.google.com/?hl=en&sl=sa&tl=en&text=[LUTE]&op=translate',1,4); -INSERT INTO languagedicts VALUES(47,12,'terms','popuphtml','https://www.spanishdict.com/translate/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(48,12,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Spanish',1,2); -INSERT INTO languagedicts VALUES(49,12,'sentences','popuphtml','https://www.deepl.com/translator#es/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(50,12,'terms','popuphtml','https://dle.rae.es/[LUTE]?m=form',1,4); -INSERT INTO languagedicts VALUES(51,12,'terms','popuphtml','https://conjugator.reverso.net/conjugation-spanish-verb-[LUTE].html',1,5); -INSERT INTO languagedicts VALUES(52,13,'terms','embeddedhtml','https://tureng.com/tr/turkce-ingilizce/[LUTE]',1,1); -INSERT INTO languagedicts VALUES(53,13,'terms','embeddedhtml','https://en.wiktionary.org/wiki/[LUTE]#Turkish',1,2); -INSERT INTO languagedicts VALUES(54,13,'sentences','popuphtml','https://www.deepl.com/translator#tr/en/[LUTE]',1,3); -INSERT INTO languagedicts VALUES(55,13,'terms','embeddedhtml','https://sozluk.gov.tr',1,4); -INSERT INTO languagedicts VALUES(56,13,'terms','embeddedhtml','https://www.verbix.com/webverbix/go.php?&D1=31&T1=[LUTE]',1,5); CREATE TABLE IF NOT EXISTS "languages" ( "LgID" INTEGER NOT NULL , "LgName" VARCHAR(40) NOT NULL , @@ -302,19 +190,6 @@ CREATE TABLE IF NOT EXISTS "languages" ( "LgParserType" VARCHAR(20) NOT NULL DEFAULT 'spacedel' , PRIMARY KEY ("LgID") ); -INSERT INTO languages VALUES(1,'Arabic','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?؟۔‎','Mr.|Mrs.|Dr.|[A-Z].|Vd.|Vds.','\u0600-\u06FF\uFE70-\uFEFC',1,1,'spacedel'); -INSERT INTO languages VALUES(2,'Classical Chinese','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?。！？','Mr.|Mrs.|Dr.|[A-Z].|Vd.|Vds.','一-龥',0,1,'classicalchinese'); -INSERT INTO languages VALUES(3,'Czech','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','Mr.|Mrs.|Dr.|[A-Z].|Vd.|Vds.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑ',0,1,'spacedel'); -INSERT INTO languages VALUES(4,'English','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','Mr.|Mrs.|Dr.|[A-Z].|Vd.|Vds.|U.S.|St.|No.|pp.|Jr.|p.m.|a.m.|Inc.|Gov.|Rep.|Ms.|Sen.|in.|Co.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑ',0,0,'spacedel'); -INSERT INTO languages VALUES(5,'French','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','[A-Z].|etc.|sens.|fin.|St.|sûr.|sol.|nom.|point.|Dr.|bout.|dos.|haut.|pp.|vol.|av.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑ',0,0,'spacedel'); -INSERT INTO languages VALUES(6,'German','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','[A-Z].|Dr.|St.|bzw.|Mio.|Co.|ca.|Mrd.|u.a.|Prof.|Nr.|Hrsg.|Chr.|II.|III.|z.B.|usw.|usf.|d.h.|e.V.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑ\u200C\u200D',0,0,'spacedel'); -INSERT INTO languages VALUES(7,'Greek','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?;','[A-Z].|[Α-Ω].|κτλ.|κλπ.|π.χ.|λ.χ.|κ.ά|δηλ.|Κος.|Κ.|Κα.|μ.Χ.|ΥΓ.|μ.μ.|π.μ.|σελ.|κεφ.|βλ.|αι.|Ε.Ε.|Δ.Σ.|Α.Ε.|Γ.Σ.|π.Χ.|τ.χλμ.|τ.μ.|κ.λπ.','α-ωΑ-ΩάόήέώύίΊΏΈΉΌΆΎϊΪϋΫΐΰ',0,1,'spacedel'); -INSERT INTO languages VALUES(8,'Hindi','´=''|`=''|’=''|‘=''|...=…|..=‥','.?!|।॥','[A-Z].|[\u0900-\u0963].|[\u0966-\u097F].|[\u200C\u200D].|है.|ए.|हैं.|ई.|ओ.|हूं.|था.|चाहिए.|म.प्र.|होगा.|थी.|स.|ए.एस.आई.|उ.प्र.|न.|ए.टी.एम.|जाएगा.|प.|हो.|ए.के.|ई.पू.|सल्ल.|मी.|सी.|ए.एस.|एम.|इ.|डी.|रजि.|पू.|टी.','a-zA-Z\u0900-\u0963\u0966-\u097F\u200C\u200D',0,1,'spacedel'); -INSERT INTO languages VALUES(9,'Japanese','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?。？！','..|→.|、.|...|「.|〜.|○○○.|『.|』.|㌶.|.....|‐.|℡.|①.|②.|③.|・・・','\p{Han}\p{Katakana}\p{Hiragana}',0,1,'japanese'); -INSERT INTO languages VALUES(10,'Russian','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','[A-Z].|[А-Я].|тыс.|млн.|руб.|ул.|млрд.|год.|им.|ст.|т.д.|гг.|др.|кв.|т.е.|ред.|все.|прим.|нас.|грн.','А-Яа-яЁё',0,1,'spacedel'); -INSERT INTO languages VALUES(11,'Sanskrit','´=''|`=''|’=''|‘=''|...=…|..=‥','.?!।॥','[\u0900-\u0963].|[\u0966-\u097F].|[A-Z].|श.|ई.|स.|अ.|ए.|उ.|च.कि.मी.|मी.|ई.पू.|इ.|p.|ऐ.|ओ.|च.|आ.|ड.|वा.','a-zA-Zāīūṣḥṃṛṭṇḍṝḹḻśṅñ\u0900-\u0963\u0966-\u097F',0,1,'spacedel'); -INSERT INTO languages VALUES(12,'Spanish','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?¡','Dr.|Dra.|Drs.|[A-Z].|Ud.|Vd.|Vds.|Sr.|Sra.|Srta.|a. C.|d.C.|Pte.|etc.|UU.|p.m.|a.m.|vs.|Jr.|pp.|St.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑ',0,0,'spacedel'); -INSERT INTO languages VALUES(13,'Turkish','´=''|`=''|’=''|‘=''|...=…|..=‥','.!?','[A-Z].|[À-Ö].|vb.|M.Ö.|Sh.|A.Ş.|T.C.|vs.|var.|Ş.|M.A.','a-zA-ZÀ-ÖØ-öø-ȳáéíóúÁÉÍÓÚñÑğĞıİöÖüÜşŞçÇ',0,1,'turkish'); CREATE TABLE textbookmarks ( "TbID" INTEGER NOT NULL, "TbTxID" INTEGER NOT NULL, diff --git a/lute/db/schema/empty.sql b/lute/db/schema/empty.sql deleted file mode 100644 index a1add6aad..000000000 --- a/lute/db/schema/empty.sql +++ /dev/null @@ -1,203 +0,0 @@ --- ------------------------------------------ --- EMPTY DB. --- Migrations tracked in _migrations, settings reset. --- Generated from 'inv db.export.empty' --- ------------------------------------------ - -PRAGMA foreign_keys=OFF; -BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "_migrations" ( - "filename" VARCHAR(255) NOT NULL , - PRIMARY KEY ("filename") -); -INSERT INTO _migrations VALUES('20230409_224327_load_statuses.sql'); -INSERT INTO _migrations VALUES('20230414_225828_add_texttokens_TokTextLC.sql'); -INSERT INTO _migrations VALUES('20230428_224656_create_wordflashmessages_table.sql'); -INSERT INTO _migrations VALUES('20230518_190000_remove_old_words_fields.sql'); -INSERT INTO _migrations VALUES('20230519_194627_add_TxDateRead.sql'); -INSERT INTO _migrations VALUES('20230621_010000_drop_texttags_table.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_01_booktags.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_02_wordtags.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_03_sentences.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_04_texttokens.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_05_texts.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_06_bookstats.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_07_termimages.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_08_wordflashmessages.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_09_wordparents.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_10_words.sql'); -INSERT INTO _migrations VALUES('20230621_224416_fk_11_books.sql'); -INSERT INTO _migrations VALUES('20230623_234104_drop_TxTitle.sql'); -INSERT INTO _migrations VALUES('20230624_182104_drop_index_TxBkIDTxOrder.sql'); -INSERT INTO _migrations VALUES('20230818_201200_add_BkWordCount.sql'); -INSERT INTO _migrations VALUES('20230819_044107_drop_texttokens.sql'); -INSERT INTO _migrations VALUES('20230819_050036_vacuum.sql'); -INSERT INTO _migrations VALUES('20230827_052154_allow_multiple_word_parents.sql'); -INSERT INTO _migrations VALUES('20231018_211236_remove_excess_texts_fields.sql'); -INSERT INTO _migrations VALUES('20231029_092851_create_migration_settings.sql'); -INSERT INTO _migrations VALUES('20231101_203811_modify_settings_schema.sql'); -CREATE TABLE IF NOT EXISTS "languages" ( - "LgID" INTEGER NOT NULL , - "LgName" VARCHAR(40) NOT NULL , - "LgDict1URI" VARCHAR(200) NOT NULL , - "LgDict2URI" VARCHAR(200) NULL , - "LgGoogleTranslateURI" VARCHAR(200) NULL , - "LgCharacterSubstitutions" VARCHAR(500) NOT NULL , - "LgRegexpSplitSentences" VARCHAR(500) NOT NULL , - "LgExceptionsSplitSentences" VARCHAR(500) NOT NULL , - "LgRegexpWordCharacters" VARCHAR(500) NOT NULL , - "LgRemoveSpaces" TINYINT NOT NULL , - "LgSplitEachChar" TINYINT NOT NULL , - "LgRightToLeft" TINYINT NOT NULL , - "LgShowRomanization" TINYINT NOT NULL DEFAULT '0' , - "LgParserType" VARCHAR(20) NOT NULL DEFAULT 'spacedel' , - PRIMARY KEY ("LgID") -); -CREATE TABLE IF NOT EXISTS "statuses" ( - "StID" INTEGER NOT NULL , - "StText" VARCHAR(20) NOT NULL , - "StAbbreviation" VARCHAR(5) NOT NULL , - PRIMARY KEY ("StID") -); -INSERT INTO statuses VALUES(0,'Unknown','?'); -INSERT INTO statuses VALUES(1,'New (1)','1'); -INSERT INTO statuses VALUES(2,'New (2)','2'); -INSERT INTO statuses VALUES(3,'Learning (3)','3'); -INSERT INTO statuses VALUES(4,'Learning (4)','4'); -INSERT INTO statuses VALUES(5,'Learned','5'); -INSERT INTO statuses VALUES(98,'Ignored','Ign'); -INSERT INTO statuses VALUES(99,'Well Known','WKn'); -CREATE TABLE IF NOT EXISTS "tags" ( - "TgID" INTEGER NOT NULL , - "TgText" VARCHAR(20) NOT NULL , - "TgComment" VARCHAR(200) NOT NULL DEFAULT '' , - PRIMARY KEY ("TgID") -); -CREATE TABLE IF NOT EXISTS "tags2" ( - "T2ID" INTEGER NOT NULL , - "T2Text" VARCHAR(20) NOT NULL , - "T2Comment" VARCHAR(200) NOT NULL DEFAULT '' , - PRIMARY KEY ("T2ID") -); -CREATE TABLE IF NOT EXISTS "booktags" ( - "BtBkID" INTEGER NOT NULL , - "BtT2ID" INTEGER NOT NULL , - PRIMARY KEY ("BtBkID", "BtT2ID"), - FOREIGN KEY("BtT2ID") REFERENCES "tags2" ("T2ID") ON UPDATE NO ACTION ON DELETE CASCADE, - FOREIGN KEY("BtBkID") REFERENCES "books" ("BkID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "wordtags" ( - "WtWoID" INTEGER NOT NULL , - "WtTgID" INTEGER NOT NULL , - PRIMARY KEY ("WtWoID", "WtTgID"), - FOREIGN KEY("WtWoID") REFERENCES "words" ("WoID") ON UPDATE NO ACTION ON DELETE CASCADE, - FOREIGN KEY("WtTgID") REFERENCES "tags" ("TgID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "sentences" ( - "SeID" INTEGER NOT NULL , - "SeTxID" INTEGER NOT NULL , - "SeOrder" SMALLINT NOT NULL , - "SeText" TEXT NULL , - PRIMARY KEY ("SeID"), - FOREIGN KEY("SeTxID") REFERENCES "texts" ("TxID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "bookstats" ( - "BkID" INTEGER NOT NULL , - "wordcount" INTEGER NULL , - "distinctterms" INTEGER NULL , - "distinctunknowns" INTEGER NULL , - "unknownpercent" INTEGER NULL , - PRIMARY KEY ("BkID"), - FOREIGN KEY("BkID") REFERENCES "books" ("BkID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "wordimages" ( - "WiID" INTEGER NOT NULL , - "WiWoID" INTEGER NOT NULL , - "WiSource" VARCHAR(500) NOT NULL , - PRIMARY KEY ("WiID"), - FOREIGN KEY("WiWoID") REFERENCES "words" ("WoID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "wordflashmessages" ( - "WfID" INTEGER NOT NULL, - "WfWoID" INTEGER NOT NULL, - "WfMessage" VARCHAR(200) NOT NULL, - PRIMARY KEY ("WfID"), - FOREIGN KEY("WfWoID") REFERENCES "words" ("WoID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "words" ( - "WoID" INTEGER NOT NULL PRIMARY KEY , - "WoLgID" INTEGER NOT NULL , - "WoText" VARCHAR(250) NOT NULL , - "WoTextLC" VARCHAR(250) NOT NULL , - "WoStatus" TINYINT NOT NULL , - "WoTranslation" VARCHAR(500) NULL , - "WoRomanization" VARCHAR(100) NULL , - "WoTokenCount" TINYINT NOT NULL DEFAULT '0' , - "WoCreated" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP , - "WoStatusChanged" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY("WoLgID") REFERENCES "languages" ("LgID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "books" ( - "BkID" INTEGER NOT NULL , - "BkLgID" INTEGER NOT NULL , - "BkTitle" VARCHAR(200) NOT NULL , - "BkSourceURI" VARCHAR(1000) NULL , - "BkArchived" TINYINT NOT NULL DEFAULT '0' , - "BkCurrentTxID" INTEGER NOT NULL DEFAULT '0' , BkWordCount INT, - PRIMARY KEY ("BkID"), - FOREIGN KEY("BkLgID") REFERENCES "languages" ("LgID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "wordparents" ( - "WpWoID" INTEGER NOT NULL , - "WpParentWoID" INTEGER NOT NULL , - FOREIGN KEY("WpParentWoID") REFERENCES "words" ("WoID") ON UPDATE NO ACTION ON DELETE CASCADE, - FOREIGN KEY("WpWoID") REFERENCES "words" ("WoID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "texts" ( - "TxID" INTEGER NOT NULL , - "TxBkID" INTEGER NOT NULL , - "TxOrder" INTEGER NOT NULL , - "TxText" TEXT NOT NULL , - TxReadDate datetime null, - PRIMARY KEY ("TxID"), - FOREIGN KEY("TxBkID") REFERENCES "books" ("BkID") ON UPDATE NO ACTION ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "settings" ( - "StKey" VARCHAR(40) NOT NULL, - "StKeyType" TEXT NOT NULL, - "StValue" TEXT NULL, - PRIMARY KEY ("StKey") -); -INSERT INTO settings VALUES('backup_enabled','user',NULL); -INSERT INTO settings VALUES('backup_auto','user','1'); -INSERT INTO settings VALUES('backup_warn','user','1'); -INSERT INTO settings VALUES('backup_dir','user',NULL); -INSERT INTO settings VALUES('backup_count','user','5'); -INSERT INTO settings VALUES('mecab_path','user',NULL); -INSERT INTO settings VALUES('custom_styles','user','/* Custom css to modify Lute''s appearance. */'); -CREATE UNIQUE INDEX "LgName" ON "languages" ("LgName"); -CREATE UNIQUE INDEX "TgText" ON "tags" ("TgText"); -CREATE UNIQUE INDEX "T2Text" ON "tags2" ("T2Text"); -CREATE INDEX "BtT2ID" ON "booktags" ("BtT2ID"); -CREATE INDEX "WtTgID" ON "wordtags" ("WtTgID"); -CREATE INDEX "SeOrder" ON "sentences" ("SeOrder"); -CREATE INDEX "SeTxID" ON "sentences" ("SeTxID"); -CREATE INDEX "WiWoID" ON "wordimages" ("WiWoID"); -CREATE INDEX "WoLgID" ON "words" ("WoLgID"); -CREATE INDEX "WoStatus" ON "words" ("WoStatus"); -CREATE INDEX "WoStatusChanged" ON "words" ("WoStatusChanged"); -CREATE INDEX "WoTextLC" ON "words" ("WoTextLC"); -CREATE UNIQUE INDEX "WoTextLCLgID" ON "words" ("WoTextLC", "WoLgID"); -CREATE INDEX "WoTokenCount" ON "words" ("WoTokenCount"); -CREATE INDEX "BkLgID" ON "books" ("BkLgID"); -CREATE UNIQUE INDEX "wordparent_pair" ON "wordparents" ("WpWoID", "WpParentWoID"); -CREATE TRIGGER trig_words_update_WoStatusChanged -AFTER UPDATE OF WoStatus ON words -FOR EACH ROW -WHEN old.WoStatus <> new.WoStatus -BEGIN - UPDATE words - SET WoStatusChanged = CURRENT_TIMESTAMP - WHERE WoID = NEW.WoID; -END; -COMMIT; diff --git a/lute/dev_api/routes.py b/lute/dev_api/routes.py index 9fe195e69..da39aaaee 100644 --- a/lute/dev_api/routes.py +++ b/lute/dev_api/routes.py @@ -47,6 +47,7 @@ def wipe_db(): def load_demo(): "Clean out everything, and load the demo." lute.db.management.delete_all_data(db.session) + lute.db.demo.set_load_demo_flag(db.session) lute.db.demo.load_demo_data(db.session) flash("demo loaded") return redirect("/", 302) diff --git a/lute/language/routes.py b/lute/language/routes.py index 0eecbd5de..294e18169 100644 --- a/lute/language/routes.py +++ b/lute/language/routes.py @@ -117,7 +117,7 @@ def new(langname): Create a new language. """ service = Service(db.session) - predefined = service.predefined_languages() + predefined = service.supported_predefined_languages() language = Language() if langname is not None: candidates = [lang for lang in predefined if lang.name == langname] @@ -163,9 +163,9 @@ def delete(langid): @bp.route("/list_predefined", methods=["GET"]) def list_predefined(): - "Show predefined languages that are not already in the db." + "Show supported predefined languages that are not already in the db." service = Service(db.session) - predefined = service.predefined_languages() + predefined = service.supported_predefined_languages() existing_langs = db.session.query(Language).all() existing_names = [l.name for l in existing_langs] new_langs = [p for p in predefined if p.name not in existing_names] diff --git a/lute/language/service.py b/lute/language/service.py index cd948e5f1..65dcd5cda 100644 --- a/lute/language/service.py +++ b/lute/language/service.py @@ -7,21 +7,49 @@ from lute.models.language import Language from lute.book.model import Book, Repository +# from lute.utils.debug_helpers import DebugTimer + class LangDef: "Language, built from language definition.yml, and .txt book files." + # Map of definition.yaml directory to the yaml.safe_load content. + # The definition files never change, and loading them takes time, so + # cache it for better unit test performance. + yaml_cache = {} + + @classmethod + def _get_loaded_yaml(cls, definition_file_path): + "Get from cache, or load it and return it." + if definition_file_path not in LangDef.yaml_cache: + with open(definition_file_path, "r", encoding="utf-8") as df: + d = yaml.safe_load(df) + LangDef.yaml_cache[definition_file_path] = d + return LangDef.yaml_cache[definition_file_path] + def __init__(self, directory): "Build from files." - self.language = self._load_lang_def(directory) - self.books = self._get_books(directory, self.language.name) + self.directory = directory + self.language_name = self._get_name(directory) + + def _get_name(self, directory): + def_file = os.path.join(directory, "definition.yaml") + d = LangDef._get_loaded_yaml(def_file) + return d["name"] + + @property + def language(self): + return self._load_lang_def(self.directory) + + @property + def books(self): + return self._get_books(self.directory, self.language_name) def _load_lang_def(self, directory): "Load from file, must exist." def_file = os.path.join(directory, "definition.yaml") - with open(def_file, "r", encoding="utf-8") as df: - d = yaml.safe_load(df) - return Language.from_dict(d) + d = LangDef._get_loaded_yaml(def_file) + return Language.from_dict(d) def _get_books(self, directory, language_name): "Get the stories." @@ -46,45 +74,42 @@ class Service: def __init__(self, session): self.session = session - self.language_from_lang_defs_cache = None + self.lang_defs_cache = self._get_langdefs_cache() - def _language_defs_path(self): - "Path to the definitions and stories." + def _get_langdefs_cache(self): + "Load cache." + # dt = DebugTimer("_get_langdefs_cache", False) + # dt.step("start") thisdir = os.path.dirname(__file__) - d = os.path.join(thisdir, "..", "db", "language_defs") - return os.path.abspath(d) - - def _load_lang_defs_cache(self): - "Cache Languages build from lang defs." - if self.language_from_lang_defs_cache is not None: - return - cache = {} - def_glob = os.path.join(self._language_defs_path(), "**", "definition.yaml") - for f in glob(def_glob): + langdefs_dir = os.path.join(thisdir, "..", "db", "language_defs") + langdefs_dir = os.path.abspath(langdefs_dir) + # dt.step("got base directory") + cache = [] + def_glob = os.path.join(langdefs_dir, "**", "definition.yaml") + def_list = glob(def_glob) + # dt.step("globbed") + def_list.sort() + for f in def_list: lang_dir, def_yaml = os.path.split(f) ld = LangDef(lang_dir) - cache[ld.language.name] = ld - self.language_from_lang_defs_cache = cache + # dt.step(f"build ld {ld.language_name}".ljust(30)) + cache.append(ld) + # dt.summary() + return cache def get_supported_defs(self): "Return supported language definitions." - self._load_lang_defs_cache() - ret = [ - {"language": ld.language, "books": ld.books} - for _, ld in self.language_from_lang_defs_cache.items() - if ld.language.is_supported - ] - ret.sort(key=lambda x: x["language"].name) + ret = [ld for ld in self.lang_defs_cache if ld.language.is_supported] + ret.sort(key=lambda x: x.language_name) return ret - def predefined_languages(self): - "Languages defined in yaml files." - return [d["language"] for d in self.get_supported_defs()] + def supported_predefined_languages(self): + "Supported Languages defined in yaml files." + return [d.language for d in self.get_supported_defs()] def get_language_def(self, lang_name): "Get a lang def and its stories." - defs = self.get_supported_defs() - ret = [d for d in defs if d["language"].name == lang_name] + ret = [ld for ld in self.lang_defs_cache if ld.language_name == lang_name] if len(ret) == 0: raise RuntimeError(f"Missing language def name {lang_name}") return ret[0] @@ -92,12 +117,15 @@ def get_language_def(self, lang_name): def load_language_def(self, lang_name): "Load a language def and its stories, save to database." load_def = self.get_language_def(lang_name) - lang = load_def["language"] + lang = load_def.language + if not lang.is_supported: + raise RuntimeError(f"{lang_name} not supported, can't be loaded.") + self.session.add(lang) self.session.commit() r = Repository(self.session) - for b in load_def["books"]: + for b in load_def.books: r.add(b) r.commit() diff --git a/lute/main.py b/lute/main.py index 4c8467197..715b1597f 100644 --- a/lute/main.py +++ b/lute/main.py @@ -17,6 +17,8 @@ from lute import __version__ from lute.app_factory import create_app from lute.config.app_config import AppConfig +from lute.db import db +from lute.db.demo import should_load_demo_data, load_demo_data logging.getLogger("waitress.queue").setLevel(logging.ERROR) logging.getLogger("natto").setLevel(logging.CRITICAL) @@ -78,6 +80,10 @@ def _start(args): config_file_path = _get_config_file_path(args.config) app = create_app(config_file_path, output_func=_print) + with app.app_context(): + if should_load_demo_data(db.session): + _print(f"Loading demo data.") + load_demo_data(db.session) close_msg = """ When you're finished reading, stop this process diff --git a/lute/utils/debug_helpers.py b/lute/utils/debug_helpers.py index a883d5a3f..fc2afc95a 100644 --- a/lute/utils/debug_helpers.py +++ b/lute/utils/debug_helpers.py @@ -53,10 +53,10 @@ def step(self, s): def summary(self): "Print final step summary." - print(f"{self.name} summary ------------------") + print(f"{self.name} summary ------------------", flush=True) for k, v in self.step_map.items(): print(f" {k}: {v:.6f}", flush=True) - print(f"end {self.name} summary --------------") + print(f"end {self.name} summary --------------", flush=True) @classmethod def clear_total_summary(cls): @@ -65,7 +65,7 @@ def clear_total_summary(cls): @classmethod def total_summary(cls): "Print final step summary." - print("global summary ------------------") + print("global summary ------------------", flush=True) for k, v in cls.global_step_map.items(): print(f" {k}: {v:.6f}", flush=True) - print("end global summary --------------") + print("end global summary --------------", flush=True) diff --git a/tasks.py b/tasks.py index 8a71ebfd3..30f59ed12 100644 --- a/tasks.py +++ b/tasks.py @@ -284,25 +284,14 @@ def full(c): # pylint: disable=unused-argument # DB tasks -@task(pre=[_ensure_test_db]) -def db_wipe(c): - """ - Wipe the data from the testing db; factory reset settings. :-) - - Can only be run on a testing db. - """ - c.run("pytest -m dbwipe") - print("ok") - - @task(pre=[_ensure_test_db]) def db_reset(c): """ - Reset the database to the demo data. + Reset the database to baseline state for new installations, with LoadDemoData system flag set. Can only be run on a testing db. """ - c.run("pytest -m dbdemoload") + c.run("pytest -m dbreset") print("ok") @@ -352,13 +341,16 @@ def db_export_baseline(c): print("quitting.") return _do_schema_export( - c, "baseline.sql", "Baseline db with demo data.", "db.export.baseline" + c, + "baseline.sql", + "Baseline db with flag to load demo data.", + "db.export.baseline", ) fname = os.path.join(_schema_dir(), "baseline.sql") print(f"Verifying {fname}") with open(fname, "r", encoding="utf-8") as f: - checkstring = "Tutorial follow-up" + checkstring = 'CREATE TABLE IF NOT EXISTS "languages"' if checkstring in f.read(): print(f'"{checkstring}" found, likely ok.') else: @@ -366,24 +358,6 @@ def db_export_baseline(c): raise RuntimeError(f'Missing "{checkstring}" in exported file.') -@task -def db_export_empty(c): - """ - Create a new empty db file from the current db. - - This assumes that the current db is in data/test_lute.db. - """ - - # Running the delete task before this one as a pre- step was - # causing problems (sqlite file not in correct state), so this - # asks the user to verify. - text = input("Have you **WIPED** the db? (y/n): ") - if text != "y": - print("quitting.") - return - _do_schema_export(c, "empty.sql", "EMPTY DB.", "db.export.empty") - - @task(help={"suffix": "suffix to add to filename."}) def db_newscript(c, suffix): # pylint: disable=unused-argument """ @@ -401,11 +375,9 @@ def db_newscript(c, suffix): # pylint: disable=unused-argument dbtasks = Collection("db") dbtasks.add_task(db_reset, "reset") -dbtasks.add_task(db_wipe, "wipe") dbtasks.add_task(db_newscript, "newscript") dbexport = Collection("export") dbexport.add_task(db_export_baseline, "baseline") -dbexport.add_task(db_export_empty, "empty") dbtasks.add_collection(dbexport) ns.add_collection(dbtasks) diff --git a/tests/conftest.py b/tests/conftest.py index 0b36e28e9..eec02d7ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,7 +108,9 @@ def _get_test_language(lang_name): if lang is not None: return lang service = Service(db.session) - lang = service.get_language_def(lang_name)["language"] + lang = service.get_language_def(lang_name).language + db.session.add(lang) + db.session.commit() return lang diff --git a/tests/features/test_rendering.py b/tests/features/test_rendering.py index 54b93bb95..df8cf87a7 100644 --- a/tests/features/test_rendering.py +++ b/tests/features/test_rendering.py @@ -7,6 +7,7 @@ from lute.db import db from lute.models.language import Language +from lute.language.service import Service as LanguageService from lute.term.model import Repository from lute.read.render.service import Service as RenderService from lute.read.service import Service @@ -27,10 +28,13 @@ @given("demo data") def given_demo_data(app_context): "Calling app_context loads the demo data." + # TODO remove this @given(parsers.parse("language {langname}")) def given_lang(langname): + svc = LanguageService(db.session) + svc.load_language_def(langname) global language # pylint: disable=global-statement lang = db.session.query(Language).filter(Language.name == langname).first() assert lang.name == langname, "sanity check" diff --git a/tests/features/test_term_import.py b/tests/features/test_term_import.py index 3cf37f4b5..b12f0f14e 100644 --- a/tests/features/test_term_import.py +++ b/tests/features/test_term_import.py @@ -11,6 +11,7 @@ from lute.db import db from lute.models.language import Language +from lute.language.service import Service as LanguageService from lute.models.term import Term from lute.models.repositories import LanguageRepository, TermRepository from lute.termimport.service import Service, BadImportFileError @@ -30,7 +31,10 @@ @given("demo data") def given_demo_data(app_context): - "Calling app_context loads the demo data." + "Load languages necessary for imports." + svc = LanguageService(db.session) + for lang in ["Spanish", "English", "Classical Chinese"]: + svc.load_language_def(lang) @given(parsers.parse("import file:\n{newcontent}")) diff --git a/tests/unit/backup/test_backup.py b/tests/unit/backup/test_backup.py index 6a1d636c0..7a0d033e2 100644 --- a/tests/unit/backup/test_backup.py +++ b/tests/unit/backup/test_backup.py @@ -10,6 +10,9 @@ from lute.backup.service import Service, BackupException, DatabaseBackupFile from lute.models.repositories import UserSettingRepository from lute.db import db +from lute.language.service import Service as LanguageService + +from tests.dbasserts import assert_record_count_equals # pylint: disable=missing-function-docstring # Test method names are pretty descriptive already. @@ -186,6 +189,13 @@ def test_warn_if_last_backup_never_happened_or_is_old(backup_settings): backup_settings.backup_warn = True backup_settings.last_backup_datetime = None service = Service(db.session) + + assert_record_count_equals("select * from books", 0, "sanity check, no books") + assert service.backup_warning(backup_settings) == "", "no warning if db empty" + + langsvc = LanguageService(db.session) + langsvc.load_language_def("English") + assert_record_count_equals("select * from books", 2, "sanity check, have books") assert service.backup_warning(backup_settings) == "Never backed up." backup_settings.last_backup_datetime = one_week_ago + 10 diff --git a/tests/unit/book/test_Repository.py b/tests/unit/book/test_Repository.py index 2c620ae63..f9c7071da 100644 --- a/tests/unit/book/test_Repository.py +++ b/tests/unit/book/test_Repository.py @@ -23,6 +23,10 @@ def fixture_book(english): Term business object with some defaults, no tags or parents. """ + if english.id is None: + db.session.add(english) + db.session.commit() + assert english.id is not None, "have english lang sanity check" b = Book() b.language_id = english.id b.title = "HELLO" @@ -32,7 +36,7 @@ def fixture_book(english): return b -def test_save_new(app_context, new_book, repo): +def test_save_new_book(app_context, new_book, repo): """ Saving a simple Book object loads the database. """ diff --git a/tests/unit/book/test_datatables.py b/tests/unit/book/test_datatables.py index e93d07fd7..45d23dab9 100644 --- a/tests/unit/book/test_datatables.py +++ b/tests/unit/book/test_datatables.py @@ -7,7 +7,7 @@ from lute.models.language import Language from lute.book.datatables import get_data_tables_list from lute.db import db -from lute.db.demo import load_demo_stories +from lute.db.demo import load_demo_data from tests.utils import make_book @@ -35,7 +35,7 @@ def test_smoke_book_datatables_query_runs(app_context, _dt_params): """ Smoke test only, ensure query runs. """ - load_demo_stories(db.session) + load_demo_data(db.session) get_data_tables_list(_dt_params, False, db.session) # print(d['data']) a = 1 @@ -46,7 +46,7 @@ def test_book_query_only_returns_supported_language_books(app_context, _dt_param """ Smoke test only, ensure query runs. """ - load_demo_stories(db.session) + load_demo_data(db.session) for lang in db.session.query(Language).all(): lang.parser_type = "unknown" db.session.add(lang) diff --git a/tests/unit/cli/test_language_term_export.py b/tests/unit/cli/test_language_term_export.py index d52fac20b..a32086418 100644 --- a/tests/unit/cli/test_language_term_export.py +++ b/tests/unit/cli/test_language_term_export.py @@ -3,15 +3,23 @@ from lute.cli.language_term_export import generate_language_file, generate_book_file from lute.models.term import Term, TermTag -from lute.models.repositories import TermRepository +from lute.models.repositories import TermRepository, LanguageRepository from lute.models.book import Book from lute.db import db -from tests.dbasserts import assert_sql_result +from lute.db.demo import load_demo_data +from tests.dbasserts import assert_sql_result, assert_record_count_equals -def test_smoke_test(app_context, tmp_path, english): +def test_language_term_export_smoke_test(app_context, tmp_path): "dump data." - t = Term(english, "the") + load_demo_data(db.session) + sql = """select * from books + where BkLgID = (select LgID from languages where LgName='English') + """ + assert_record_count_equals(sql, 2, "have books") + langrepo = LanguageRepository(db.session) + eng = langrepo.find_by_name("English") + t = Term(eng, "the") t.translation = "article" t.add_term_tag(TermTag("a")) t.add_term_tag(TermTag("b")) diff --git a/tests/unit/db/test_demo.py b/tests/unit/db/test_demo.py index f511fdcea..51951d7c6 100644 --- a/tests/unit/db/test_demo.py +++ b/tests/unit/db/test_demo.py @@ -1,11 +1,27 @@ """ Tests for managing the demo data. + +Prior to https://github.com/LuteOrg/lute-v3/issues/534, the baseline +db had languages and stories pre-loaded. This created a lot of db +file thrash whenever that data changed. + +In the new setup, the baseline db only contains a flag, "LoadDemoData" +(true/false), which is initially set to True. When the app first +starts up, if that flag is True, it loads the demo data, and sets +"LoadDemoData" to False, and "IsDemoData" to True. + +If the LoadDemoData flag is set, the demo data is loaded from the +startup scripts (devstart and lute.main) + """ from sqlalchemy import text import pytest from lute.db import db from lute.db.demo import ( + set_load_demo_flag, + remove_load_demo_flag, + should_load_demo_data, contains_demo_data, remove_flag, delete_demo_data, @@ -16,25 +32,73 @@ from tests.dbasserts import assert_record_count_equals, assert_sql_result -def test_new_db_is_demo(app_context): +# ======================================== +# See notes at top of file re these tests. +# ======================================== + + +def test_new_db_doesnt_contain_anything(app_context): "New db created from the baseline has the demo flag set." - assert contains_demo_data(db.session) is True, "new db contains demo." + assert should_load_demo_data(db.session) is True, "has LoadDemoData flag." + assert contains_demo_data(db.session) is False, "no demo data." + + +def test_empty_db_not_loaded_if_load_flag_not_set(app_context): + "Even if it's empty, nothing happens." + remove_load_demo_flag(db.session) + assert contains_demo_data(db.session) is False, "no demo data." + assert_record_count_equals("select * from languages", 0, "empty") + load_demo_data(db.session) + assert contains_demo_data(db.session) is False, "no demo data." + assert should_load_demo_data(db.session) is False, "still no reload." + assert_record_count_equals("select * from languages", 0, "still empty") + + +def test_smoke_test_load_demo_works(app_context): + "Wipe everything, but set the flag and then start." + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) + assert contains_demo_data(db.session) is True, "demo loaded." + assert tutorial_book_id(db.session) > 0, "Have tutorial" + assert should_load_demo_data(db.session) is False, "loaded once, don't reload." + + +def test_load_not_run_if_data_exists_even_if_flag_is_set(app_context): + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) + assert tutorial_book_id(db.session) > 0, "Have tutorial" + assert should_load_demo_data(db.session) is False, "loaded once, don't reload." + + remove_flag(db.session) + set_load_demo_flag(db.session) + assert should_load_demo_data(db.session) is True, "should re-reload demo data." + load_demo_data(db.session) # if this works, it didn't throw :-P + assert should_load_demo_data(db.session) is False, "already loaded once." def test_removing_flag_means_not_demo(app_context): "Unsetting the flag means the db is not a demo." + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) + assert contains_demo_data(db.session) is True, "demo loaded." remove_flag(db.session) - assert contains_demo_data(db.session) is False, "not a demo." + assert contains_demo_data(db.session) is False, "not a demo now." def test_wiping_db_clears_flag(app_context): "No longer a demo if the demo is wiped out!" + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) + assert contains_demo_data(db.session) is True, "demo loaded." delete_demo_data(db.session) assert contains_demo_data(db.session) is False, "not a demo." def test_wipe_db_only_works_if_flag_is_set(app_context): "Can only wipe a demo db!" + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) + assert contains_demo_data(db.session) is True, "demo loaded." remove_flag(db.session) with pytest.raises(Exception): delete_demo_data(db.session) @@ -42,6 +106,8 @@ def test_wipe_db_only_works_if_flag_is_set(app_context): def test_tutorial_id_returned_if_present(app_context): "Sanity check." + assert should_load_demo_data(db.session) is True, "should reload demo data." + load_demo_data(db.session) assert tutorial_book_id(db.session) > 0, "have tutorial" sql = 'update books set bktitle = "xxTutorial" where bktitle = "Tutorial"' @@ -61,27 +127,15 @@ def test_tutorial_id_returned_if_present(app_context): # Loading. -@pytest.mark.dbdemoload -def test_load_demo_loads_language_yaml_files(app_context): +@pytest.mark.dbreset +def test_rebaseline(app_context): """ - All data is loaded, spot check some. - This test is also used from "inv db.reset" in tasks.py (see .pytest.ini). """ - delete_demo_data(db.session) assert contains_demo_data(db.session) is False, "not a demo." assert_record_count_equals("languages", 0, "wiped out") - load_demo_data(db.session) - assert contains_demo_data(db.session) is True, "demo loaded" - checks = [ - "select * from languages where LgName = 'English'", - "select * from books where BkTitle = 'Tutorial'", - ] - for c in checks: - assert_record_count_equals(c, 1, c + " returned 1") - # Wipe out all user settings!!! When user installs and first # starts up, the user settings need to be loaded with values from # _their_ config and environment. @@ -89,8 +143,10 @@ def test_load_demo_loads_language_yaml_files(app_context): db.session.execute(text(sql)) db.session.commit() + set_load_demo_flag(db.session) + sql = "select stkeytype, stkey, stvalue from settings" - assert_sql_result(sql, ["system; IsDemoData; 1"], "only this key is set.") + assert_sql_result(sql, ["system; LoadDemoData; 1"], "only this key is set.") @pytest.fixture(name="_restore_japanese_parser") diff --git a/tests/unit/db/test_management.py b/tests/unit/db/test_management.py index 514b7834c..4cec3ed21 100644 --- a/tests/unit/db/test_management.py +++ b/tests/unit/db/test_management.py @@ -13,12 +13,9 @@ from tests.dbasserts import assert_record_count_equals -@pytest.mark.dbwipe def test_wiping_db_clears_out_all_tables(app_context): """ DB is wiped clean if requested ... settings are left! - - This test is also used from /tasks.py; see .pytest.ini. """ old_user_settings = db.session.query(UserSetting).all() diff --git a/tests/unit/language/test_service.py b/tests/unit/language/test_service.py index 4cce0e42b..43b664871 100644 --- a/tests/unit/language/test_service.py +++ b/tests/unit/language/test_service.py @@ -1,32 +1,42 @@ """ -Read service tests. +Language service tests. """ -# from lute.db import db from lute.language.service import Service from lute.db import db from tests.dbasserts import assert_sql_result +from lute.utils.debug_helpers import DebugTimer def test_get_all_lang_defs(app_context): "Can get all predefined languages." service = Service(db.session) defs = service.get_supported_defs() - engs = [d for d in defs if d["language"].name == "English"] + engs = [d for d in defs if d.language.name == "English"] assert len(engs) == 1, "have english" eng = engs[0] - assert len(eng["books"]) == 2, "tutorial and follow-up" - titles = [b.title for b in eng["books"]] + assert len(eng.books) == 2, "tutorial and follow-up" + titles = [b.title for b in eng.books] titles.sort() assert titles == ["Tutorial", "Tutorial follow-up"], "book titles" -def test_get_language_def(): +def test_supported_predefined_languages(app_context): + "Get supported lang names" + service = Service(db.session) + predefs = service.supported_predefined_languages() + assert len(predefs) > 1, "Have predefined" + langnames = [lang.name for lang in predefs] + assert "English" in langnames, "Have English" + assert "French" in langnames, "Have French" + + +def test_get_language_def(app_context): """ Smoke test, can load a new language from yaml definition. """ service = Service(db.session) - lang = service.get_language_def("English")["language"] + lang = service.get_language_def("English").language assert lang.name == "English" assert lang.show_romanization is False, "uses default" @@ -46,30 +56,37 @@ def test_get_language_def(): assert actual == expected, "dictionaries" -def test_load_def_loads_lang_and_stories(empty_db): +def test_load_def_loads_lang_and_stories(app_context): "Can load a language." story_sql = "select bktitle from books order by BkTitle" lang_sql = "select LgName from languages" assert_sql_result(lang_sql, [], "no langs") assert_sql_result(story_sql, [], "nothing loaded") + dt = DebugTimer("Loading", False) + dt.step("start") service = Service(db.session) + dt.step("Service()") lang_id = service.load_language_def("English") + dt.step("load_language_def") + dt.summary() + assert lang_id > 0, "ID returned, used for filtering" assert_sql_result(lang_sql, ["English"], "eng loaded") assert_sql_result(story_sql, ["Tutorial", "Tutorial follow-up"], "stories loaded") -def test_load_all_defs_loads_lang_and_stories(empty_db): +def test_load_all_defs_loads_lang_and_stories(app_context): "Smoke test, load everything." story_sql = "select bktitle from books" lang_sql = "select LgName from languages" assert_sql_result(lang_sql, [], "no langs") assert_sql_result(story_sql, [], "nothing loaded") + db.session.flush() service = Service(db.session) defs = service.get_supported_defs() - langnames = [d["language"].name for d in defs] + langnames = [d.language.name for d in defs] for n in langnames: lang_id = service.load_language_def(n) assert lang_id > 0, "Loaded" diff --git a/tests/unit/models/test_Language.py b/tests/unit/models/test_Language.py index 95dbfc26a..8535d689c 100644 --- a/tests/unit/models/test_Language.py +++ b/tests/unit/models/test_Language.py @@ -5,6 +5,7 @@ """ from lute.db import db +from lute.db.demo import load_demo_data from lute.models.language import Language from lute.models.repositories import LanguageRepository from tests.dbasserts import assert_sql_result @@ -15,6 +16,7 @@ def test_demo_has_preloaded_languages(app_context): When users get the initial demo, it has English, French, etc, pre-defined. """ + load_demo_data(db.session) sql = """ select LgName from languages @@ -42,6 +44,10 @@ def test_can_find_lang_by_name(app_context): """ Returns lang if found, or None """ + lang = Language() + lang.name = "English" + db.session.add(lang) + db.session.commit() repo = LanguageRepository(db.session) e = repo.find_by_name("English") assert e.name == "English", "case match" @@ -64,6 +70,7 @@ def test_language_word_char_regex_returns_python_compatible_regex(app_context): u0600-u06FFuFE70-uFEFC (where u = backslash-u) """ + load_demo_data(db.session) repo = LanguageRepository(db.session) a = repo.find_by_name("Arabic") assert a.word_characters == r"\u0600-\u06FF\uFE70-\uFEFC" @@ -75,6 +82,7 @@ def test_lang_to_dict_from_dict_returns_same_thing(app_context): A dictionary is used as the intermediary form, so the same language should return the same data. """ + load_demo_data(db.session) repo = LanguageRepository(db.session) e = repo.find_by_name("English") e_dict = e.to_dict() diff --git a/tests/unit/utils/test_formutils.py b/tests/unit/utils/test_formutils.py index 1785d8418..0fe4202a6 100644 --- a/tests/unit/utils/test_formutils.py +++ b/tests/unit/utils/test_formutils.py @@ -7,7 +7,7 @@ from lute.models.repositories import UserSettingRepository -def test_language_choices(app_context): +def test_language_choices(app_context, spanish, english): "Gets all languages." choices = language_choices(db.session) assert choices[0][1] == "-", "- at the top" @@ -15,7 +15,13 @@ def test_language_choices(app_context): assert "Spanish" in langnames, "have Spanish" -def test_valid_current_language_id(app_context): +def test_language_choices_if_only_single_language_exists(app_context, spanish): + "Gets all languages." + choices = language_choices(db.session) + assert choices[0][1] == "Spanish", "sole choice possible" + + +def test_valid_current_language_id(app_context, spanish, english): "Sanity check only." repo = UserSettingRepository(db.session) repo.set_value("current_language_id", 9999) From 2b41e2faa247f322aeba778a51979a60b99be3d9 Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Mon, 9 Dec 2024 11:26:23 -0600 Subject: [PATCH 2/4] Resetting db automatically exports new baseline. --- tasks.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/tasks.py b/tasks.py index 30f59ed12..52cbcbf57 100644 --- a/tasks.py +++ b/tasks.py @@ -284,17 +284,6 @@ def full(c): # pylint: disable=unused-argument # DB tasks -@task(pre=[_ensure_test_db]) -def db_reset(c): - """ - Reset the database to baseline state for new installations, with LoadDemoData system flag set. - - Can only be run on a testing db. - """ - c.run("pytest -m dbreset") - print("ok") - - def _schema_dir(): "Return full path to schema dir." thisdir = os.path.dirname(os.path.realpath(__file__)) @@ -330,16 +319,8 @@ def _do_schema_export(c, destfile, header_notes, taskname): @task def db_export_baseline(c): """ - Reset the db, and create a new baseline db file from the current db. + Create a new baseline db file from the current db. """ - - # Running the delete task before this one as a pre- step was - # causing problems (sqlite file not in correct state), so this - # asks the user to verify. - text = input("Have you reset the db? (y/n): ") - if text != "y": - print("quitting.") - return _do_schema_export( c, "baseline.sql", @@ -358,6 +339,17 @@ def db_export_baseline(c): raise RuntimeError(f'Missing "{checkstring}" in exported file.') +@task(pre=[_ensure_test_db], post=[db_export_baseline]) +def db_reset(c): + """ + Reset the database to baseline state for new installations, with LoadDemoData system flag set. + + Can only be run on a testing db. + """ + c.run("pytest -m dbreset") + print("\nok, exporting baseline.sql.\n") + + @task(help={"suffix": "suffix to add to filename."}) def db_newscript(c, suffix): # pylint: disable=unused-argument """ From 085f5f77b82d127a789c5d71147bb025ba6256b8 Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Mon, 9 Dec 2024 20:43:12 -0600 Subject: [PATCH 3/4] Change db.demo to service. --- lute/app_factory.py | 18 +- lute/db/demo.py | 323 ++++++++++---------- lute/dev_api/routes.py | 13 +- lute/main.py | 7 +- tests/conftest.py | 2 +- tests/unit/book/test_datatables.py | 8 +- tests/unit/cli/test_language_term_export.py | 5 +- tests/unit/db/test_demo.py | 124 ++++---- tests/unit/models/test_Language.py | 11 +- 9 files changed, 257 insertions(+), 254 deletions(-) diff --git a/lute/app_factory.py b/lute/app_factory.py index 7d6893ff8..71a70da59 100644 --- a/lute/app_factory.py +++ b/lute/app_factory.py @@ -28,7 +28,7 @@ from lute.db.management import add_default_user_settings from lute.db.data_cleanup import clean_data from lute.backup.service import Service as BackupService -import lute.db.demo +from lute.db.demo import Service as DemoService import lute.utils.formutils from lute.parse.registry import init_parser_plugins, supported_parsers @@ -128,7 +128,8 @@ def inject_menu_bar_vars(): @app.route("/") def index(): - is_production = not lute.db.demo.contains_demo_data(db.session) + demosvc = DemoService(db.session) + is_production = not demosvc.contains_demo_data() us_repo = UserSettingRepository(db.session) bkp_settings = us_repo.get_backup_settings() @@ -153,13 +154,14 @@ def index(): and warning_msg != "" ) + demosvc = DemoService(db.session) response = make_response( render_template( "index.html", hide_homelink=True, dbname=app_config.dbname, datapath=app_config.datapath, - tutorial_book_id=lute.db.demo.tutorial_book_id(db.session), + tutorial_book_id=demosvc.tutorial_book_id(), have_books=have_books, have_languages=have_languages, language_choices=language_choices, @@ -181,8 +183,9 @@ def refresh_all_stats(): @app.route("/wipe_database") def wipe_db(): - if lute.db.demo.contains_demo_data(db.session): - lute.db.demo.delete_demo_data(db.session) + demosvc = DemoService(db.session) + if demosvc.contains_demo_data(): + demosvc.delete_demo_data() msg = """ The database has been wiped clean. Have fun!

(Lute has automatically enabled backups -- @@ -193,8 +196,9 @@ def wipe_db(): @app.route("/remove_demo_flag") def remove_demo(): - if lute.db.demo.contains_demo_data(db.session): - lute.db.demo.remove_flag(db.session) + demosvc = DemoService(db.session) + if demosvc.contains_demo_data(): + demosvc.remove_flag() msg = """ Demo mode deactivated. Have fun!

(Lute has automatically enabled backups -- diff --git a/lute/db/demo.py b/lute/db/demo.py index 0608a0bad..b02eaa27b 100644 --- a/lute/db/demo.py +++ b/lute/db/demo.py @@ -9,174 +9,167 @@ """ from sqlalchemy import text -from lute.language.service import Service +from lute.language.service import Service as LanguageService from lute.book.model import Repository from lute.book.stats import Service as StatsService from lute.models.repositories import SystemSettingRepository, LanguageRepository import lute.db.management -def _demo_languages(): - """ - Demo languages to be loaded for new users. - Also loaded during tests. - """ - return [ - "Arabic", - "Classical Chinese", - "Czech", - "English", - "French", - "German", - "Greek", - "Hindi", - "Japanese", - "Russian", - "Sanskrit", - "Spanish", - "Turkish", - ] - - -def set_load_demo_flag(session): - "Set the flag." - repo = SystemSettingRepository(session) - repo.set_value("LoadDemoData", True) - session.commit() - - -def remove_load_demo_flag(session): - "Set the flag." - repo = SystemSettingRepository(session) - repo.delete_key("LoadDemoData") - session.commit() - - -def _flag_exists(session, flagname): - "True if flag exists, else false." - repo = SystemSettingRepository(session) - return repo.key_exists(flagname) - - -def should_load_demo_data(session): - return _flag_exists(session, "LoadDemoData") - - -def contains_demo_data(session): - return _flag_exists(session, "IsDemoData") - - -def remove_flag(session): - """ - Remove IsDemoData setting. - """ - if not contains_demo_data(session): - raise RuntimeError("Can't delete non-demo data.") - - repo = SystemSettingRepository(session) - repo.delete_key("IsDemoData") - session.commit() - - -def tutorial_book_id(session): - """ - Return the book id of the tutorial. - """ - if not contains_demo_data(session): - return None - sql = """select BkID from books - inner join languages on LgID = BkLgID - where LgName = 'English' and BkTitle = 'Tutorial' - """ - r = session.execute(text(sql)).first() - if r is None: - return None - return int(r[0]) - - -def delete_demo_data(session): - """ - If this is a demo, wipe everything. - """ - if not contains_demo_data(session): - raise RuntimeError("Can't delete non-demo data.") - remove_flag(session) - lute.db.management.delete_all_data(session) - - -# Loading demo data. - - -def load_demo_languages(session): - """ - Load selected predefined languages, if they're supported. - - This method will also be called during acceptance tests, so it's public. - """ - demo_langs = _demo_languages() - service = Service(session) - langs = [service.get_language_def(langname).language for langname in demo_langs] - supported = [lang for lang in langs if lang.is_supported] - for lang in supported: - session.add(lang) - session.commit() - - -def load_demo_stories(session): - "Load the stories for any languages already loaded." - demo_langs = _demo_languages() - service = Service(session) - langdefs = [service.get_language_def(langname) for langname in demo_langs] - - langrepo = LanguageRepository(session) - langdefs = [ - d - for d in langdefs - if d.language.is_supported - and langrepo.find_by_name(d.language.name) is not None - ] - - r = Repository(session) - for d in langdefs: - for b in d.books: - r.add(b) - r.commit() - - repo = SystemSettingRepository(session) - repo.set_value("IsDemoData", True) - session.commit() - - svc = StatsService(session) - svc.refresh_stats() - - -def _db_has_data(session): - "True of the db contains any language data." - sql = "select LgID from languages limit 1" - r = session.execute(text(sql)).first() - return r is not None - - -def load_demo_data(session): - """ - Load the data. - """ - if _db_has_data(session): - remove_load_demo_flag(session) - return - - repo = SystemSettingRepository(session) - do_load = repo.get_value("LoadDemoData") - if do_load is None: - # Only load if flag is explicitly set. - return - - do_load = bool(int(do_load)) - if not do_load: - return - - load_demo_languages(session) - load_demo_stories(session) - remove_load_demo_flag(session) - repo.set_value("IsDemoData", True) - session.commit() +class Service: + "Demo database service." + + def __init__(self, session): + self.session = session + + def _demo_languages(self): + """ + Demo languages to be loaded for new users. + Also loaded during tests. + """ + return [ + "Arabic", + "Classical Chinese", + "Czech", + "English", + "French", + "German", + "Greek", + "Hindi", + "Japanese", + "Russian", + "Sanskrit", + "Spanish", + "Turkish", + ] + + def set_load_demo_flag(self): + "Set the flag." + repo = SystemSettingRepository(self.session) + repo.set_value("LoadDemoData", True) + self.session.commit() + + def remove_load_demo_flag(self): + "Set the flag." + repo = SystemSettingRepository(self.session) + repo.delete_key("LoadDemoData") + self.session.commit() + + def _flag_exists(self, flagname): + "True if flag exists, else false." + repo = SystemSettingRepository(self.session) + return repo.key_exists(flagname) + + def should_load_demo_data(self): + return self._flag_exists("LoadDemoData") + + def contains_demo_data(self): + return self._flag_exists("IsDemoData") + + def remove_flag(self): + """ + Remove IsDemoData setting. + """ + if not self.contains_demo_data(): + raise RuntimeError("Can't delete non-demo data.") + + repo = SystemSettingRepository(self.session) + repo.delete_key("IsDemoData") + self.session.commit() + + def tutorial_book_id(self): + """ + Return the book id of the tutorial. + """ + if not self.contains_demo_data(): + return None + sql = """select BkID from books + inner join languages on LgID = BkLgID + where LgName = 'English' and BkTitle = 'Tutorial' + """ + r = self.session.execute(text(sql)).first() + if r is None: + return None + return int(r[0]) + + def delete_demo_data(self): + """ + If this is a demo, wipe everything. + """ + if not self.contains_demo_data(): + raise RuntimeError("Can't delete non-demo data.") + self.remove_flag() + lute.db.management.delete_all_data(self.session) + + # Loading demo data. + + def load_demo_languages(self): + """ + Load selected predefined languages, if they're supported. + + This method will also be called during acceptance tests, so it's public. + """ + demo_langs = self._demo_languages() + service = LanguageService(self.session) + langs = [service.get_language_def(langname).language for langname in demo_langs] + supported = [lang for lang in langs if lang.is_supported] + for lang in supported: + self.session.add(lang) + self.session.commit() + + def load_demo_stories(self): + "Load the stories for any languages already loaded." + demo_langs = self._demo_languages() + service = LanguageService(self.session) + langdefs = [service.get_language_def(langname) for langname in demo_langs] + + langrepo = LanguageRepository(self.session) + langdefs = [ + d + for d in langdefs + if d.language.is_supported + and langrepo.find_by_name(d.language.name) is not None + ] + + r = Repository(self.session) + for d in langdefs: + for b in d.books: + r.add(b) + r.commit() + + repo = SystemSettingRepository(self.session) + repo.set_value("IsDemoData", True) + self.session.commit() + + svc = StatsService(self.session) + svc.refresh_stats() + + def _db_has_data(self): + "True of the db contains any language data." + sql = "select LgID from languages limit 1" + r = self.session.execute(text(sql)).first() + return r is not None + + def load_demo_data(self): + """ + Load the data. + """ + if self._db_has_data(): + self.remove_load_demo_flag() + return + + repo = SystemSettingRepository(self.session) + do_load = repo.get_value("LoadDemoData") + if do_load is None: + # Only load if flag is explicitly set. + return + + do_load = bool(int(do_load)) + if not do_load: + return + + self.load_demo_languages() + self.load_demo_stories() + self.remove_load_demo_flag() + repo.set_value("IsDemoData", True) + self.session.commit() diff --git a/lute/dev_api/routes.py b/lute/dev_api/routes.py index da39aaaee..7d3444df9 100644 --- a/lute/dev_api/routes.py +++ b/lute/dev_api/routes.py @@ -22,7 +22,7 @@ import lute.parse.registry from lute.db import db import lute.db.management -import lute.db.demo +from lute.db.demo import Service as DemoService bp = Blueprint("dev_api", __name__, url_prefix="/dev_api") @@ -47,8 +47,9 @@ def wipe_db(): def load_demo(): "Clean out everything, and load the demo." lute.db.management.delete_all_data(db.session) - lute.db.demo.set_load_demo_flag(db.session) - lute.db.demo.load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.set_load_demo_flag() + demosvc.load_demo_data() flash("demo loaded") return redirect("/", 302) @@ -57,7 +58,8 @@ def load_demo(): def load_demo_languages(): "Clean out everything, and load the demo langs with dummy dictionaries." lute.db.management.delete_all_data(db.session) - lute.db.demo.load_demo_languages(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_languages() langs = db.session.query(Language).all() for lang in langs: d = lang.dictionaries[0] @@ -71,7 +73,8 @@ def load_demo_languages(): @bp.route("/load_demo_stories", methods=["GET"]) def load_demo_stories(): "Stories only. No db wipe." - lute.db.demo.load_demo_stories(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_stories() flash("stories loaded") return redirect("/", 302) diff --git a/lute/main.py b/lute/main.py index 715b1597f..39ba1d9a3 100644 --- a/lute/main.py +++ b/lute/main.py @@ -18,7 +18,7 @@ from lute.app_factory import create_app from lute.config.app_config import AppConfig from lute.db import db -from lute.db.demo import should_load_demo_data, load_demo_data +from lute.db.demo import Service as DemoService logging.getLogger("waitress.queue").setLevel(logging.ERROR) logging.getLogger("natto").setLevel(logging.CRITICAL) @@ -81,9 +81,10 @@ def _start(args): config_file_path = _get_config_file_path(args.config) app = create_app(config_file_path, output_func=_print) with app.app_context(): - if should_load_demo_data(db.session): + demosvc = DemoService(db.session) + if demosvc.should_load_demo_data(): _print(f"Loading demo data.") - load_demo_data(db.session) + demosvc.load_demo_data() close_msg = """ When you're finished reading, stop this process diff --git a/tests/conftest.py b/tests/conftest.py index eec02d7ca..1822e342b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,8 @@ from lute.config.app_config import AppConfig from lute.db import db +import lute.db.management from lute.language.service import Service -import lute.db.demo from lute.app_factory import create_app from lute.models.language import Language diff --git a/tests/unit/book/test_datatables.py b/tests/unit/book/test_datatables.py index 45d23dab9..20abe58ae 100644 --- a/tests/unit/book/test_datatables.py +++ b/tests/unit/book/test_datatables.py @@ -7,7 +7,7 @@ from lute.models.language import Language from lute.book.datatables import get_data_tables_list from lute.db import db -from lute.db.demo import load_demo_data +from lute.db.demo import Service as DemoService from tests.utils import make_book @@ -35,7 +35,8 @@ def test_smoke_book_datatables_query_runs(app_context, _dt_params): """ Smoke test only, ensure query runs. """ - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() get_data_tables_list(_dt_params, False, db.session) # print(d['data']) a = 1 @@ -46,7 +47,8 @@ def test_book_query_only_returns_supported_language_books(app_context, _dt_param """ Smoke test only, ensure query runs. """ - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() for lang in db.session.query(Language).all(): lang.parser_type = "unknown" db.session.add(lang) diff --git a/tests/unit/cli/test_language_term_export.py b/tests/unit/cli/test_language_term_export.py index a32086418..19ebbc055 100644 --- a/tests/unit/cli/test_language_term_export.py +++ b/tests/unit/cli/test_language_term_export.py @@ -6,13 +6,14 @@ from lute.models.repositories import TermRepository, LanguageRepository from lute.models.book import Book from lute.db import db -from lute.db.demo import load_demo_data +from lute.db.demo import Service as DemoService from tests.dbasserts import assert_sql_result, assert_record_count_equals def test_language_term_export_smoke_test(app_context, tmp_path): "dump data." - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() sql = """select * from books where BkLgID = (select LgID from languages where LgName='English') """ diff --git a/tests/unit/db/test_demo.py b/tests/unit/db/test_demo.py index 51951d7c6..62ca46d41 100644 --- a/tests/unit/db/test_demo.py +++ b/tests/unit/db/test_demo.py @@ -18,16 +18,7 @@ from sqlalchemy import text import pytest from lute.db import db -from lute.db.demo import ( - set_load_demo_flag, - remove_load_demo_flag, - should_load_demo_data, - contains_demo_data, - remove_flag, - delete_demo_data, - tutorial_book_id, - load_demo_data, -) +from lute.db.demo import Service import lute.parse.registry from tests.dbasserts import assert_record_count_equals, assert_sql_result @@ -37,103 +28,108 @@ # ======================================== -def test_new_db_doesnt_contain_anything(app_context): +@pytest.fixture(name="service") +def service(app_context): + return Service(db.session) + + +def test_new_db_doesnt_contain_anything(service): "New db created from the baseline has the demo flag set." - assert should_load_demo_data(db.session) is True, "has LoadDemoData flag." - assert contains_demo_data(db.session) is False, "no demo data." + assert service.should_load_demo_data() is True, "has LoadDemoData flag." + assert service.contains_demo_data() is False, "no demo data." -def test_empty_db_not_loaded_if_load_flag_not_set(app_context): +def test_empty_db_not_loaded_if_load_flag_not_set(service): "Even if it's empty, nothing happens." - remove_load_demo_flag(db.session) - assert contains_demo_data(db.session) is False, "no demo data." + service.remove_load_demo_flag() + assert service.contains_demo_data() is False, "no demo data." assert_record_count_equals("select * from languages", 0, "empty") - load_demo_data(db.session) - assert contains_demo_data(db.session) is False, "no demo data." - assert should_load_demo_data(db.session) is False, "still no reload." + service.load_demo_data() + assert service.contains_demo_data() is False, "no demo data." + assert service.should_load_demo_data() is False, "still no reload." assert_record_count_equals("select * from languages", 0, "still empty") -def test_smoke_test_load_demo_works(app_context): +def test_smoke_test_load_demo_works(service): "Wipe everything, but set the flag and then start." - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert contains_demo_data(db.session) is True, "demo loaded." - assert tutorial_book_id(db.session) > 0, "Have tutorial" - assert should_load_demo_data(db.session) is False, "loaded once, don't reload." + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.contains_demo_data() is True, "demo loaded." + assert service.tutorial_book_id() > 0, "Have tutorial" + assert service.should_load_demo_data() is False, "loaded once, don't reload." -def test_load_not_run_if_data_exists_even_if_flag_is_set(app_context): - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert tutorial_book_id(db.session) > 0, "Have tutorial" - assert should_load_demo_data(db.session) is False, "loaded once, don't reload." +def test_load_not_run_if_data_exists_even_if_flag_is_set(service): + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.tutorial_book_id() > 0, "Have tutorial" + assert service.should_load_demo_data() is False, "loaded once, don't reload." - remove_flag(db.session) - set_load_demo_flag(db.session) - assert should_load_demo_data(db.session) is True, "should re-reload demo data." - load_demo_data(db.session) # if this works, it didn't throw :-P - assert should_load_demo_data(db.session) is False, "already loaded once." + service.remove_flag() + service.set_load_demo_flag() + assert service.should_load_demo_data() is True, "should re-reload demo data." + service.load_demo_data() # if this works, it didn't throw :-P + assert service.should_load_demo_data() is False, "already loaded once." -def test_removing_flag_means_not_demo(app_context): +def test_removing_flag_means_not_demo(service): "Unsetting the flag means the db is not a demo." - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert contains_demo_data(db.session) is True, "demo loaded." - remove_flag(db.session) - assert contains_demo_data(db.session) is False, "not a demo now." + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.contains_demo_data() is True, "demo loaded." + service.remove_flag() + assert service.contains_demo_data() is False, "not a demo now." -def test_wiping_db_clears_flag(app_context): +def test_wiping_db_clears_flag(service): "No longer a demo if the demo is wiped out!" - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert contains_demo_data(db.session) is True, "demo loaded." - delete_demo_data(db.session) - assert contains_demo_data(db.session) is False, "not a demo." + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.contains_demo_data() is True, "demo loaded." + service.delete_demo_data() + assert service.contains_demo_data() is False, "not a demo." -def test_wipe_db_only_works_if_flag_is_set(app_context): +def test_wipe_db_only_works_if_flag_is_set(service): "Can only wipe a demo db!" - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert contains_demo_data(db.session) is True, "demo loaded." - remove_flag(db.session) + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.contains_demo_data() is True, "demo loaded." + service.remove_flag() with pytest.raises(Exception): - delete_demo_data(db.session) + service.delete_demo_data() -def test_tutorial_id_returned_if_present(app_context): +def test_tutorial_id_returned_if_present(service): "Sanity check." - assert should_load_demo_data(db.session) is True, "should reload demo data." - load_demo_data(db.session) - assert tutorial_book_id(db.session) > 0, "have tutorial" + assert service.should_load_demo_data() is True, "should reload demo data." + service.load_demo_data() + assert service.tutorial_book_id() > 0, "have tutorial" sql = 'update books set bktitle = "xxTutorial" where bktitle = "Tutorial"' db.session.execute(text(sql)) db.session.commit() - assert tutorial_book_id(db.session) is None, "no tutorial" + assert service.tutorial_book_id() is None, "no tutorial" sql = 'update books set bktitle = "Tutorial" where bktitle = "xxTutorial"' db.session.execute(text(sql)) db.session.commit() - assert tutorial_book_id(db.session) > 0, "have tutorial again" + assert service.tutorial_book_id() > 0, "have tutorial again" - delete_demo_data(db.session) - assert tutorial_book_id(db.session) is None, "no tutorial" + service.delete_demo_data() + assert service.tutorial_book_id() is None, "no tutorial" # Loading. @pytest.mark.dbreset -def test_rebaseline(app_context): +def test_rebaseline(service): """ This test is also used from "inv db.reset" in tasks.py (see .pytest.ini). """ - assert contains_demo_data(db.session) is False, "not a demo." + assert service.contains_demo_data() is False, "not a demo." assert_record_count_equals("languages", 0, "wiped out") # Wipe out all user settings!!! When user installs and first @@ -143,7 +139,7 @@ def test_rebaseline(app_context): db.session.execute(text(sql)) db.session.commit() - set_load_demo_flag(db.session) + service.set_load_demo_flag() sql = "select stkeytype, stkey, stvalue from settings" assert_sql_result(sql, ["system; LoadDemoData; 1"], "only this key is set.") diff --git a/tests/unit/models/test_Language.py b/tests/unit/models/test_Language.py index 8535d689c..30b7075bd 100644 --- a/tests/unit/models/test_Language.py +++ b/tests/unit/models/test_Language.py @@ -5,7 +5,7 @@ """ from lute.db import db -from lute.db.demo import load_demo_data +from lute.db.demo import Service as DemoService from lute.models.language import Language from lute.models.repositories import LanguageRepository from tests.dbasserts import assert_sql_result @@ -16,7 +16,8 @@ def test_demo_has_preloaded_languages(app_context): When users get the initial demo, it has English, French, etc, pre-defined. """ - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() sql = """ select LgName from languages @@ -70,7 +71,8 @@ def test_language_word_char_regex_returns_python_compatible_regex(app_context): u0600-u06FFuFE70-uFEFC (where u = backslash-u) """ - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() repo = LanguageRepository(db.session) a = repo.find_by_name("Arabic") assert a.word_characters == r"\u0600-\u06FF\uFE70-\uFEFC" @@ -82,7 +84,8 @@ def test_lang_to_dict_from_dict_returns_same_thing(app_context): A dictionary is used as the intermediary form, so the same language should return the same data. """ - load_demo_data(db.session) + demosvc = DemoService(db.session) + demosvc.load_demo_data() repo = LanguageRepository(db.session) e = repo.find_by_name("English") e_dict = e.to_dict() From c716f03c10da27ebb4dc3edc0e22d0a24c6a68de Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Mon, 9 Dec 2024 20:49:01 -0600 Subject: [PATCH 4/4] Fix lint. --- lute/language/service.py | 2 +- lute/main.py | 2 +- tests/unit/db/test_demo.py | 3 ++- tests/unit/language/test_service.py | 2 +- tests/unit/utils/test_formutils.py | 2 ++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lute/language/service.py b/lute/language/service.py index 65dcd5cda..cb9f43d4c 100644 --- a/lute/language/service.py +++ b/lute/language/service.py @@ -90,7 +90,7 @@ def _get_langdefs_cache(self): # dt.step("globbed") def_list.sort() for f in def_list: - lang_dir, def_yaml = os.path.split(f) + lang_dir, _ = os.path.split(f) ld = LangDef(lang_dir) # dt.step(f"build ld {ld.language_name}".ljust(30)) cache.append(ld) diff --git a/lute/main.py b/lute/main.py index 39ba1d9a3..e79680b9b 100644 --- a/lute/main.py +++ b/lute/main.py @@ -83,7 +83,7 @@ def _start(args): with app.app_context(): demosvc = DemoService(db.session) if demosvc.should_load_demo_data(): - _print(f"Loading demo data.") + _print("Loading demo data.") demosvc.load_demo_data() close_msg = """ diff --git a/tests/unit/db/test_demo.py b/tests/unit/db/test_demo.py index 62ca46d41..c0f1743c8 100644 --- a/tests/unit/db/test_demo.py +++ b/tests/unit/db/test_demo.py @@ -29,7 +29,7 @@ @pytest.fixture(name="service") -def service(app_context): +def _service(app_context): return Service(db.session) @@ -60,6 +60,7 @@ def test_smoke_test_load_demo_works(service): def test_load_not_run_if_data_exists_even_if_flag_is_set(service): + "Just in case." assert service.should_load_demo_data() is True, "should reload demo data." service.load_demo_data() assert service.tutorial_book_id() > 0, "Have tutorial" diff --git a/tests/unit/language/test_service.py b/tests/unit/language/test_service.py index 43b664871..2f9268d43 100644 --- a/tests/unit/language/test_service.py +++ b/tests/unit/language/test_service.py @@ -4,8 +4,8 @@ from lute.language.service import Service from lute.db import db -from tests.dbasserts import assert_sql_result from lute.utils.debug_helpers import DebugTimer +from tests.dbasserts import assert_sql_result def test_get_all_lang_defs(app_context): diff --git a/tests/unit/utils/test_formutils.py b/tests/unit/utils/test_formutils.py index 0fe4202a6..f79069096 100644 --- a/tests/unit/utils/test_formutils.py +++ b/tests/unit/utils/test_formutils.py @@ -6,6 +6,8 @@ from lute.utils.formutils import language_choices, valid_current_language_id from lute.models.repositories import UserSettingRepository +# pylint: disable=unused-argument + def test_language_choices(app_context, spanish, english): "Gets all languages."