From be9c2ba98dd29c774182ce81bfababd86a67fe14 Mon Sep 17 00:00:00 2001 From: Arihant Jain Date: Tue, 16 Apr 2019 22:16:35 -0700 Subject: [PATCH 1/5] added files --- .gitignore | 6 + classroomScrape.js | 84 +++++ package-lock.json | 829 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 27 ++ webscraper.js | 151 +++++++++ 5 files changed, 1097 insertions(+) create mode 100644 .gitignore create mode 100644 classroomScrape.js create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 webscraper.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a12baca --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +node_modules.zip +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/classroomScrape.js b/classroomScrape.js new file mode 100644 index 0000000..f871e96 --- /dev/null +++ b/classroomScrape.js @@ -0,0 +1,84 @@ +"use strict"; + +const request = require("request"); +const moment = require("moment"); + +const currentTerm = "19S"; +const mainURL = + "https://www.registrar.ucla.edu/desktopmodules/ClassRoomSearch/api/webapi/GetClassroomItems"; +const classroomURL = + "https://www.registrar.ucla.edu/desktopmodules/ClassRoomSearch/api/webapi/GetCalendarEvents"; + +function processClassroomJson(buildingClassroom) { + let toReturn = { + building: buildingClassroom.value.slice(0, 8), + room: buildingClassroom.value.substr(9), + buildingName: buildingClassroom.label.slice(0, 8).trim(), + roomName: buildingClassroom.label.substr(9).trim() + }; + return toReturn; +} + +function processClassTimeObject(buildingArray, classNameTime) { + if (classNameTime.enroll_total === 0) return buildingArray; + + let processedObject = { + day: "", + start: "", + end: "" + }; + + if (classNameTime.start.substr(0, 9) != "2014-11-1") { + processedObject.day = "Varies"; + processedObject.start = classNameTime.start.substr(0, 5); + processedObject.end = classNameTime.end.substr(0, 5); + } else { + let startTime = moment(classNameTime.start); + let endTime = moment(classNameTime.end); + + processedObject.day = startTime.format("d"); + processedObject.start = startTime.format("HH:mm"); + processedObject.end = endTime.format("HH:mm"); + } + buildingArray.push(processedObject); + return buildingArray; +} + +function getClassroomTimes(buildingRoomObject) { + let options = { + method: "GET", + url: classroomURL, + qs: { + term: currentTerm, + building: buildingRoomObject.building, + room: buildingRoomObject.room + }, + json: true, + callback: function(error, response, body) { + if (error) throw new Error(error); + + body = body.reduce(processClassTimeObject, []); + + let finalClassroomObject = { + building: buildingRoomObject.buildingName, + room: buildingRoomObject.roomName, + classTimes: body + }; + console.log(finalClassroomObject); + } + }; + request(options); +} + +let options = { + method: "GET", + url: mainURL, + json: true, + callback: function(error, response, body) { + console.log("got classrooms"); + let classrooms = body.map(processClassroomJson); + classrooms.forEach(getClassroomTimes); + } +}; + +request(options); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..0c5d27e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,829 @@ +{ + "name": "backend-v2", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "abab": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz", + "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==" + }, + "acorn": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", + "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==" + }, + "acorn-globals": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.0.tgz", + "integrity": "sha512-hMtHj3s5RnuhvHPowpBYvJVj3rAar82JiDQHvGs1zO0l10ocX/xEdBShNHTJaboucJUsScghp74pH3s7EnHHQw==", + "requires": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, + "acorn-walk": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.1.1.tgz", + "integrity": "sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw==" + }, + "ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "body-parser": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } + }, + "browser-process-hrtime": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz", + "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==" + }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cssom": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.6.tgz", + "integrity": "sha512-DtUeseGk9/GBW0hl0vVPpU22iHL6YB5BUX7ml1hB+GMpo0NX5G4voX3kdWiMSEguFtcW3Vh3djqNF4aIe6ne0A==" + }, + "cssstyle": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.2.2.tgz", + "integrity": "sha512-43wY3kl1CVQSvL7wUY1qXkxVGkStjpkDmVjiIKX8R97uhajy8Bybay78uOtqvh7Q5GK75dNPfW0geWjE6qQQow==", + "requires": { + "cssom": "0.3.x" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "requires": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "requires": { + "webidl-conversions": "^4.0.2" + } + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "escodegen": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.1.tgz", + "integrity": "sha512-JwiqFD9KdGVVpeuRa68yU3zZnBEOcPs0nKW7wZzXky8Z7tffdYUHbe11bPCV5jYlK6DVdKLWLm0f5I/QlL0Kmw==", + "requires": { + "esprima": "^3.1.3", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + } + }, + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + }, + "estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "requires": { + "whatwg-encoding": "^1.0.1" + } + }, + "http-errors": { + "version": "1.6.3", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "iconv-lite": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsdom": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-14.0.0.tgz", + "integrity": "sha512-/VkyPmdtbwqpJSkwDx3YyJ3U1oawYNB/h5z8vTUZGAzjtu2OHTeFRfnJqyMHsJ5Cyes23trOmvUpM1GfHH1leA==", + "requires": { + "abab": "^2.0.0", + "acorn": "^6.0.4", + "acorn-globals": "^4.3.0", + "array-equal": "^1.0.0", + "cssom": "^0.3.4", + "cssstyle": "^1.1.1", + "data-urls": "^1.1.0", + "domexception": "^1.0.1", + "escodegen": "^1.11.0", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.0.9", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.5", + "saxes": "^3.1.5", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.5.0", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.0.1", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^6.1.2", + "xml-name-validator": "^3.0.0" + }, + "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "mime-db": { + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", + "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + }, + "mime-types": { + "version": "2.1.21", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", + "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "requires": { + "mime-db": "~1.37.0" + } + }, + "moment": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + }, + "moment-timezone": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz", + "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==", + "requires": { + "moment": ">= 2.9.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "nwsapi": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.1.3.tgz", + "integrity": "sha512-RowAaJGEgYXEZfQ7tvvdtAQUKPyTR6T6wNu0fwlNsGQYr/h3yQc6oI8WnVZh3Y/Sylwc+dtAlvPqfFZjhTyk3A==" + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + } + }, + "parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, + "psl": { + "version": "1.1.31", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", + "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "raw-body": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "request-promise-core": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz", + "integrity": "sha512-UHYyq1MO8GsefGEt7EprS8UrXsm1TxEvFUX1IMTuSLU2Rh7fTIdFtl8xD7JiEYiWU2dl+NYAjCTksTehQUxPag==", + "requires": { + "lodash": "^4.17.11" + } + }, + "request-promise-native": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.7.tgz", + "integrity": "sha512-rIMnbBdgNViL37nZ1b3L/VfPOpSi0TqVDQPAvO6U14lMzOLrt5nilxCQqtDKhZeDiW0/hkCXGoQjhgJd/tCh6w==", + "requires": { + "request-promise-core": "1.1.2", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "saxes": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.9.tgz", + "integrity": "sha512-FZeKhJglhJHk7eWG5YM0z46VHmI3KJpMBAQm3xa9meDvd+wevB5GuBB0wc0exPInZiBBHqi00DbS8AcvCGCFMw==", + "requires": { + "xmlchars": "^1.3.1" + } + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, + "symbol-tree": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz", + "integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY=" + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "requires": { + "punycode": "^2.1.0" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "type-is": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "w3c-hr-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz", + "integrity": "sha1-gqwr/2PZUOqeMYmlimViX+3xkEU=", + "requires": { + "browser-process-hrtime": "^0.1.2" + } + }, + "w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "requires": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "requires": { + "iconv-lite": "0.4.24" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } + } + }, + "whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "whatwg-url": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", + "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + }, + "ws": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.2.tgz", + "integrity": "sha512-rfUqzvz0WxmSXtJpPMX2EeASXabOrSMk1ruMOV3JBTBjo4ac2lDjGGsbQSyxj8Odhw5fBib8ZKEjDNvgouNKYw==", + "requires": { + "async-limiter": "~1.0.0" + } + }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "xmlchars": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-1.3.1.tgz", + "integrity": "sha512-tGkGJkN8XqCod7OT+EvGYK5Z4SfDQGD30zAa58OcnAa0RRWgzUEK72tkXhsX1FZd+rgnhRxFtmO+ihkp8LHSkw==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..7dbf09d --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "backend-v2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/aseem191/backend-v2.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/aseem191/backend-v2/issues" + }, + "homepage": "https://github.com/aseem191/backend-v2#readme", + "dependencies": { + "body-parser": "^1.18.3", + "jsdom": "^14.0.0", + "moment": "^2.24.0", + "moment-timezone": "^0.5.23", + "request": "^2.88.0" + } +} diff --git a/webscraper.js b/webscraper.js new file mode 100644 index 0000000..ad30d60 --- /dev/null +++ b/webscraper.js @@ -0,0 +1,151 @@ +"use strict"; + +const moment = require("moment-timezone"); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; +const request = require("request"); +/* +JSON Format { + "Room Details": "Sproul Study Room 110E (max 4 people)", + "Time": "11:00pm-midnight on Sun, Mar 03", + "Link": "https://www.orl.ucla.edu/reserve?type=sproulstudy&duration=60&date=2019-03-03&roomid=3584&start=1551682800&stop=1551686400" +} +*/ + +const meetingRooms = [ + "deneve", + "hedrick", + "hedrickstudy", + "hedrickmusic", + "movement", + "music", + "rieber", + "sproulmusic", + "sproulstudy" +]; + +const durations = ["60", "120"]; + +var now = moment().tz("America/Los_Angeles"); +const formatString = "YYYY-MM-DD"; + +let next21DaysStrings = []; + +let numSaturdays = 0; + +for (var i = 0; numSaturdays < 3; i++) { + next21DaysStrings[i] = now.format(formatString); + + if (now.day() == 6) numSaturdays++; + now.add(1, "days"); +} + +let totalRequests = 0; +let totalData = []; + +for ( + var meetingRoomIndex = 0; + meetingRoomIndex < meetingRooms.length; + meetingRoomIndex++ +) { + for ( + var durationIndex = 0; + durationIndex < durations.length; + durationIndex++ + ) { + for (var dayIndex = 0; dayIndex < next21DaysStrings.length; dayIndex++) { + ++totalRequests; + sendRequest(meetingRoomIndex, durationIndex, dayIndex); + } + } +} + +function finishRequest(oneRow) { + totalRequests--; + console.log(totalRequests); + totalData = totalData.concat(oneRow); + if (totalRequests == 0) { + request.post( + { + url: + "http://studysmart-env-2.dqiv29pdi2.us-east-1.elasticbeanstalk.com/studyinfo", + headers: { "content-type": "application/json" }, + body: JSON.stringify(totalData) + }, + function(err, response, body) { + console.log(response.body); + new Promise(resolve => setTimeout(resolve, 10000)); + console.log(JSON.stringify(response.body)); + } + ); + + console.log(totalData); + } +} + +function sendRequest(meetingRoomIndex, durationIndex, dayIndex) { + let options = { + method: "GET", + url: "http://reslife.ucla.edu/reserve", + qs: { + type: meetingRooms[meetingRoomIndex], + duration: durations[durationIndex], + date: next21DaysStrings[dayIndex], + partial: "" + }, + headers: {}, + //timeout: 20000, + callback: function(error, response, body) { + if (error) { + sendRequest(meetingRoomIndex, durationIndex, dayIndex); + return; + } else { + const dom = new JSDOM(body); + + let data = []; + let columns = dom.window.document.querySelectorAll(".col-md-6"); + //Get all links so that you can redirect user to registration page directly + let links = dom.window.document.getElementsByTagName("a"); + let filteredLinks = []; + //Need to filter all links so that we only save the ones with the class mentioned below + for (k = 0; k < links.length; k++) { + if ( + links[k].getAttribute("class") == + "btn btn-sm btn-block btn-default btn-select" + ) { + filteredLinks.push(links[k]); + } + } + //Counter for filtered links + var k = 0; + //Start at 2 because first value is not relevant + //Increment by 2 because every pair of 2 elements gives us "room details" and "time" + + for (var i = 2; i < columns.length; i += 2) { + //Create JS object and push required elements + var obj = new Object(); + obj["Room Details"] = columns[i].textContent.trim(); + obj["Time"] = columns[i + 1].textContent.trim(); + obj["Link"] = filteredLinks[k].getAttribute("href"); + k++; + //Push + data.push(obj); + } + //console.log(data); + finishRequest(data); + } + } + }; + setTimeout(o => request(o), 0, options); +} + +/* +for (let i = 0; i < result.length; i++) { + request.post({ url: "http://studysmart-env-2.dqiv29pdi2.us-east-1.elasticbeanstalk.com/studyinfo", headers: { 'content-type': 'application/json' }, body: JSON.stringify(result[i]) }, function (err, response, body) { + console.log(response.body) + new Promise((resolve) => setTimeout(resolve, 10000)); + //console.log(JSON.stringify(response.body)) + }) + // postRequest(result[i]); + } + */ From dee3b3268ea6b14464112d074bf82980981d0aee Mon Sep 17 00:00:00 2001 From: Andrew Zeff Date: Tue, 23 Apr 2019 00:48:35 -0700 Subject: [PATCH 2/5] Started busy times --- classroomScrape.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/classroomScrape.js b/classroomScrape.js index f871e96..c6f04a9 100644 --- a/classroomScrape.js +++ b/classroomScrape.js @@ -70,6 +70,11 @@ function getClassroomTimes(buildingRoomObject) { request(options); } +function getBusyTimes(buildingRoomObject) +{ + +} + let options = { method: "GET", url: mainURL, From b7273661f1c8d5f779e9acc49579378a08bfd940 Mon Sep 17 00:00:00 2001 From: Andrew Zeff Date: Tue, 28 May 2019 17:31:36 -0700 Subject: [PATCH 3/5] finished inverse scraper...sorry for the delay --- classroomScrape.js | 65 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/classroomScrape.js b/classroomScrape.js index c6f04a9..30d195a 100644 --- a/classroomScrape.js +++ b/classroomScrape.js @@ -10,13 +10,12 @@ const classroomURL = "https://www.registrar.ucla.edu/desktopmodules/ClassRoomSearch/api/webapi/GetCalendarEvents"; function processClassroomJson(buildingClassroom) { - let toReturn = { + return { building: buildingClassroom.value.slice(0, 8), room: buildingClassroom.value.substr(9), buildingName: buildingClassroom.label.slice(0, 8).trim(), roomName: buildingClassroom.label.substr(9).trim() }; - return toReturn; } function processClassTimeObject(buildingArray, classNameTime) { @@ -44,6 +43,8 @@ function processClassTimeObject(buildingArray, classNameTime) { return buildingArray; } +// let classroomTimes = []; + function getClassroomTimes(buildingRoomObject) { let options = { method: "GET", @@ -65,14 +66,63 @@ function getClassroomTimes(buildingRoomObject) { classTimes: body }; console.log(finalClassroomObject); + console.log(getAvailableTimes(finalClassroomObject)); + //getAvailableTimes(finalClassroomObject); + //classroomTimes.push(finalClassroomObject); } }; request(options); } -function getBusyTimes(buildingRoomObject) +function tomins(timeString) { + let timeObj = stoiParser(timeString); + return timeObj['hour'] * 60 + timeObj['minutes']; +} + +function stoiParser(timeString) +{ + return { + 'hour': parseInt(timeString.substring(0, 2) ), + 'minutes': parseInt(timeString.substring(3) ) + }; +} + + + + +function getAvailableTimes(classroomTimeObject) +{ + let inverseObject = {}; + inverseObject.building = classroomTimeObject.building; + inverseObject.room = classroomTimeObject.room; + inverseObject.classTimes = []; + let timesByDay = {'1': [], '2': [], '3': [], '4': [], '5': [], 'Varies':[]}; + let inverseTimesByDay = {'1': [], '2': [], '3': [], '4': [], '5': [], 'Varies':[]} + for (let timeSlot of classroomTimeObject.classTimes) { + timesByDay[timeSlot.day].push(timeSlot); + } + for(let d in timesByDay) + { + let s = '08:00'; + for(let slot of timesByDay[d]) + { + if(tomins(slot['start']) - tomins(s) > 15){ + inverseTimesByDay[d].push({'day': d, 'start': s, 'end': slot['start']}); + } + s = slot['end']; + } + inverseTimesByDay[d].push({'day': d, 'start': s, 'end': '22:00'}); + } + for(let d in inverseTimesByDay) + { + for(let slot of inverseTimesByDay[d]) + { + inverseObject.classTimes.push({'day':d, 'start':slot['start'], 'end':slot['end']}); + } + } + return inverseObject; } let options = { @@ -80,9 +130,14 @@ let options = { url: mainURL, json: true, callback: function(error, response, body) { + if (error) { + console.log("error getting classrooms", error); + return; + } console.log("got classrooms"); - let classrooms = body.map(processClassroomJson); - classrooms.forEach(getClassroomTimes); + let classrooms = body.map(processClassroomJson); + classrooms.forEach(getClassroomTimes); + //classrooms.forEach(getAvailableTimes); } }; From 64a838428f6d03ed12a893739cf2c171b3fa973e Mon Sep 17 00:00:00 2001 From: Andrew Zeff Date: Tue, 28 May 2019 21:40:14 -0700 Subject: [PATCH 4/5] set up for the database --- classroomScrape.js | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/classroomScrape.js b/classroomScrape.js index 30d195a..579dd51 100644 --- a/classroomScrape.js +++ b/classroomScrape.js @@ -1,4 +1,7 @@ "use strict"; +var AWS = require("aws-sdk"); +AWS.config.update({ region: "us-east-1" }); +var docClient = new AWS.DynamoDB.DocumentClient(); const request = require("request"); const moment = require("moment"); @@ -65,9 +68,23 @@ function getClassroomTimes(buildingRoomObject) { room: buildingRoomObject.roomName, classTimes: body }; - console.log(finalClassroomObject); - console.log(getAvailableTimes(finalClassroomObject)); - //getAvailableTimes(finalClassroomObject); + //console.log(finalClassroomObject); + // console.log(getAvailableTimes(finalClassroomObject)); + finalClassroomObject = getAvailableTimes(finalClassroomObject) + let params = { + TableName: "study_info", + Item: finalClassroomObject + }; + console.log(params); + docClient.put(params, function(err, data) { + if (err) { + throw err; + console.log("error"); + } + console.log("successfully put item"); + console.log(params); + }); + //getAvailableTimes(finalClassroomObject); //classroomTimes.push(finalClassroomObject); } }; @@ -97,6 +114,7 @@ function getAvailableTimes(classroomTimeObject) inverseObject.building = classroomTimeObject.building; inverseObject.room = classroomTimeObject.room; inverseObject.classTimes = []; + inverseObject.class_key = ""; let timesByDay = {'1': [], '2': [], '3': [], '4': [], '5': [], 'Varies':[]}; let inverseTimesByDay = {'1': [], '2': [], '3': [], '4': [], '5': [], 'Varies':[]} for (let timeSlot of classroomTimeObject.classTimes) { @@ -119,7 +137,8 @@ function getAvailableTimes(classroomTimeObject) { for(let slot of inverseTimesByDay[d]) { - inverseObject.classTimes.push({'day':d, 'start':slot['start'], 'end':slot['end']}); + inverseObject.classTimes.push({'day':d, 'start':tomins(slot['start']), 'end':tomins(slot['end'])}); + inverseObject.class_key = inverseObject.building + inverseObject.room; } } return inverseObject; @@ -138,6 +157,7 @@ let options = { let classrooms = body.map(processClassroomJson); classrooms.forEach(getClassroomTimes); //classrooms.forEach(getAvailableTimes); + } }; From e2c7e332c00dfe119cf881c0333d0fff33f9d1dc Mon Sep 17 00:00:00 2001 From: Arihant Jain Date: Tue, 28 May 2019 22:18:25 -0700 Subject: [PATCH 5/5] fixed table name --- classroomScrape.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classroomScrape.js b/classroomScrape.js index 579dd51..d470e42 100644 --- a/classroomScrape.js +++ b/classroomScrape.js @@ -72,7 +72,7 @@ function getClassroomTimes(buildingRoomObject) { // console.log(getAvailableTimes(finalClassroomObject)); finalClassroomObject = getAvailableTimes(finalClassroomObject) let params = { - TableName: "study_info", + TableName: "class_scrape_table", Item: finalClassroomObject }; console.log(params);