diff --git a/NOTES.MD b/NOTES.MD new file mode 100644 index 000000000..9762151ab --- /dev/null +++ b/NOTES.MD @@ -0,0 +1,349 @@ +# Web Server + +Software that: +- accepts HTTP requests (e.g. GET requests for HTML pages), and +- returns responses (e.g. HTML pages). +Examples: +- GET requests for getting data from RESTful API endpoints, images within web pages, and +- POST requests to upload data to the server (e.g. a form submission or file upload). +- PUT +- DELETE +- HEAD same as GET, but does not return anything in the message body. The returned metadata should be identical to info returned in a GET request. +- ping 1.1.1.1 (Internet's fastest DNS (Domain Name System) address ) +- datatrace -n 1.1.1.1 traceroute 1.1.1.1 + +## Networking Protocols +Protocol: established code of procedures between client and servers handling requests and responses. + +Data packets are encapulated from the ground up (starting with the HTTP header and ending with the Ethernet Header) + +### Network Stack / Data Encapulation Hierarchy +|-----------------------| +| Ethernet Header | Routing on the LAN Physical/Data ling +| Network Access Layer | Access physical media +|-----------------------| +| IP (Address) Header | Routing on the Internet How Data Is Transferred +| Network/Internet Layer| Defines datagram/ +|-----------------------| +| TCP Header / UDP | Data integrity Reliability +| Transport Layer | End-to-end deliver +|-----------------------| +| HTTP Header | Deals with web data Formats req & resp +| Application Layer | +|-----------------------| +|

Hello World!

| Actual data +|_______________________| + +### Ethernet +Sends data and listens at the same time to make sure it is identical. If it isn't it means someone else is sending at the same time and the data will become corrupt. So, it wait 7.6 + random seconds and resends. If too many try sending at the same time the ethernet fails/overload (packet storm). + +### TCP Protocol (Transmission Control Protocol) +- Reliable +- Uses Three-Way Handshake +- Breaks packet into smaller packets + +Client/Initiator Server / Listener +---------------------------------- ----------------------------------------- +connect() listen() + -----SYN-----> TCB initialized to SYN-RECEIVED STATE + +Success code returned by connect() <---SYN-ACK--- + + -----ACK-----> TCB transitions to Established State^^^ + + Completion: Data packets exchanged + +^^^ If the ACK was never recieved the server has another set for losts packets. + +### UDP (User Datagram Protocol) Protocol (vs TCP) +- Not as reliable +- Less protocols and more lost packets +- Ability to send significantly more packets than TCP +- No handshake or acknowledgement + + + +## Unix Sockets +- Higher level example of a socket is socket.io API +- Low level sockets API is what this project will implement + - supports both TCP & UDP and IPv4 and IPv6 + +### Files in Unix: "Everything in Unix is a file." +File Descriptors (fd): Integer value associated with an open file. +FILE *fp = fopen("text.txt", "w"); +printf("%d, fp); // this would print an integer value + +File Descriptor Global File Table Inode Table +--------------- ----------------------- ------------------- +0 read-only, offset: 0 +1 write-only, offset: 0 /dev/pts22 +2 +3 read-write, offset: 12 /path/myfile.txt +4 read-write, offset: 8 /path/myfile3.txt + +Caveat: File can refer to: +- network connection (shared file) +- pipe +- terminal +- actual file + +### Unix-Style Sockets API System Calls + +Client +--------- +socket() +connect() +write() +read() + +Server +--------- +socket() +bind() net.c 79 +listen() net.c 100 +accept() +read() +write() + +- socket() creates a new socket (the fd is stored to an int) +- bind() bind IP address (associated with a network card) +- send() write data +- recv() read data + + +#### Initializing a Socket + +```c +#include +int sockfd = socket (...) // initialize a socket (file descriptor) +send(sockfd, response, responseLength, 0); // 0 is a flag +recv(sockfd, request, requestBufferSize - 1, 0); // 0 is a flag +``` + +P1 P2 + _____ ______ +fd _____ shared file ______ fd + + + +Services by Port Number +- 7: echo +- 21: ftp (File Transfer Protocol) +- 23: telnet (TCP connection with a server & issue raw HTTP requests) + telnet www.example.com 80 +- 25: smtp (Simple Mail Transfer) +- 80: http / www +- 6000: x11 (X Window System -- a unix based gui) + + + + +## HTTP Protocol +- HTTP responsible for formatting requests and responses between client and servers +- HTTP does *not* handle how data is transfered. +- HTTP does *not* hand reliability + +### How to Format HTTP Packets + +#### Format of a GET Request +Syntax: +HTTP Method /URL Protocol/Version ** /URL w/o .com +Host: Host of target URL +Blank line indicating the end of the Request use \n (even if there is no body a blank line is required) + +Example: +GET /example HTTP/1.1 +Host: lambdaschool.com +Connection: close +X-Header: whatever + + +GET /pub/WWW/TheProject.html HTTP/1.1 +Host: www.w3.org + + +#### Format of a POST Request +Syntax: +POST request-URI HTTP-version +Content-Type: mime-type +Content-Length: number-of-bytes +(other optional request headers) + +(URL-encoded query string) + +Example: +POST /magazines HTTP/1.1 +Host: example.gov.au +Accept: application/json, text/javascript + +#### Format of a DELETE Request +DELETE /magazines HTTP/1.1 +Host: example.gov.au +Accept: application/json, text/javascript + +#### Format of a HEAD Request +HEAD /magazines HTTP/1.1 +Host: example.gov.au +Accept: application/json, text/javascript + +### Format of a Response +Syntax: +Protocol/Version Status Code +Date: Timestamp when response was created +Connection: close (set to open if more info will be sent) +Content-Length: number of bytes in the body (w/o blank line) +Content-Type: MIME (Multipurpose Internet Mail Extensions) type of the content of the body to be rendered (text/html, text/plain, text/css, image/jpg, image/gif, image/png, application/json, application/javascript, multipart/form-data) + +Data being sent back + +#### Example: 200 Response +HTTP/1.1 200 OK +Date: Wed Dec 20 13:05:11 PST 2017 +Connection: close +Content-Length: 12345 +Content-Type: text/html + +Lambda + +#### Example: 404 Response +HTTP/1.1 404 Not Found +Date: Wed Dec 20 13:05:11 PST 2017 +Connection: close +Content-Length: 13 +Content-Type: text/plain + +404 Not Found + +send_response() response_length send() sprintf() strlen() sprintf() time() localtime() time.h + + +## MISC + +### sscanf & spacing +- useful for parsing incoming request from a web browser +- sscanf is scanf for strings +- Request and responses have specific spacing + - e.g.: GET /example HTTP/1.1 + HTTP/1.1 404 Not Found +- %20 is the ASCII Hex code for space. Used for space in URL (no additional spaces allowed) +- JS equivalent to .split(" ") +- C compiler will concatenate char * that are next to each other, e.g., char *s = "my" "string" + +```c +#include + +/* +GET /foobar HTTP/1.1 +Host: www.example.com +Connection: closecon +X-Header: whatever +*/ + +int main(void) +{ + // s holds the HTTP request + char *s = "GET /foobar HTTP/1.1\nHost: www.example.com\nConnection: close\nX-Header: whatever"; + + char method[200]; + char path[8192]; + + sscanf(s, "%s %s", method, path); + + printf("method: \"%s\"\n", method); + printf("path: \"%s\"\n", path); +} +``` + +### sprintf & Content Length +- useful for building outgoing response to the server +- use char *body to save body. +- int length = strlen(body); +- use %s + +```c +#include +#include +​ +int main(void) +{ + char response[500000]; +​ + char *body = "

Hello, world!

"; + int length = strlen(body); +​ + sprintf(response, "HTTP/1.1 200 OK\n" + "Content-Type: text/html\n" + "Content-Length: %d\n" + "Connection: close\n" + "\n" + "%s", + length, body + ); +​ + printf("%s", response); // send() +} +``` + +## Cache + +Get Requests +GET /index.html +GET /index.html +GET /style.css +GET /foo.gif +GET /style.css +GET /index.html + +Cache(3) Disk + + + +head tail + +bar -> foo -> style +style -> bar -> foo + +|||||||HASHTABLE||||||| +index <-> style <-> bar + +Add a cache miss to head of list O(1) +Remove LRU from end of list O(1) +Remove LRU from hashtable O(1) + +Item in cache? O(1) + +## Demo Deep Copy: Lifetime of Pointer malloc + +### Deep Copy: type char +// Go with Option B +// malloc & strcpy + +struct foo { + char *p; +} + +struct foo *alloc_foo(char *x) { + struct foo *f = malloc(sizeof *f); + f->p = x; + return f; +} + +struct foo *f1(void) { + + // Option A + char s[] = "Hello!"; + + // Option B: malloc on the heap + char *t = malloc(7); + strcpy(t, "Hello!"); + + + struct foo *q = alloc_foo(s); // Option A + struct foo *q = alloc_foo(t); // Option B + return q; +} + +### Deep Copy type void + (void) content; + ce->content = malloc(content_length); + memcpy(ce->content, content, content_length); diff --git a/PROJECT.MD b/PROJECT.MD new file mode 100644 index 000000000..0d1207189 --- /dev/null +++ b/PROJECT.MD @@ -0,0 +1,30 @@ +Todo: + +* HTTP request parser +* HTTP response builder +* LRU cache + * Doubly linked list (some functionality provided) + * Use existing hashtable functionality (below) + +Already done: + +* `net.h` and `net.c` contain low-level networking code +* `mime.h` and `mime.c` contains functionality for determining the MIME type of a file +* `file.h` and `file.c` contains handy file-reading code that you may want to utilize, namely the `file_load()` and `file_free()` functions for reading file data and deallocating file data, respectively (or you could just perform these operations manually as well) +* `hashtable.h` and `hashtable.c` contain an implementation of a hashtable (this one is a bit more complicated than what you built in the Hashtables sprint) +* `llist.h` and `llist.c` contain an implementation of a doubly-linked list (used solely by the hashable--you don't need it) +* `cache.h` and `cache.c` are where you will implement the LRU cache functionality for days 3 and 4 + + +## Start at main (in server.c) +- Start listening for connections +- When a connection arrives, receive the request data. +- Parse the reqest data. +- Build response data. +- Send response data. +- Close the connection and wait for the next one. + +## Create Function & Inverse Function: use reverse order +- cache_create & cache_free: chache_free frees the cache, thus cache_create must malloc it. + + diff --git a/src/cache.c b/src/cache.c index c72975cdd..49985bf45 100644 --- a/src/cache.c +++ b/src/cache.c @@ -9,9 +9,25 @@ */ struct cache_entry *alloc_entry(char *path, char *content_type, void *content, int content_length) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// + struct cache_entry * ce = malloc(sizeof *ce); // = sizeof (struct cache_entry) + + // Deep Copy + // Fleeting Pointer: use malloc & strcpy (not just: ce->path, path) + ce->path = malloc(strlen(path)+1); // + for '\0' + strcpy(ce->path, path); + + ce->content_type = malloc(strlen(content_type)+1); + strcpy(ce->content_type, content_type); + + // content is type void, but we have content_length to compensate + ce->content = malloc(content_length); + memcpy(ce->content, content, content_length); + + ce->content_length = content_length; + + ce->prev = ce->next = NULL; + + return ce; } /** @@ -19,9 +35,10 @@ struct cache_entry *alloc_entry(char *path, char *content_type, void *content, i */ void free_entry(struct cache_entry *entry) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// + free(entry->path); + free(entry->content_type); + free(entry->content); + free(entry); } /** @@ -91,9 +108,14 @@ struct cache_entry *dllist_remove_tail(struct cache *cache) */ struct cache *cache_create(int max_size, int hashsize) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// + + struct cache *cache = malloc(sizeof *cache); + + cache->index = hashtable_create(hashsize, NULL); // NULL for default hash function + cache->head = cache->tail = NULL; //Doubly-Linked List + cache->max_size = max_size; // Max entries + cache->cur_size = 0; // Curr No. of entries + return cache; } void cache_free(struct cache *cache) @@ -122,9 +144,28 @@ void cache_free(struct cache *cache) */ void cache_put(struct cache *cache, char *path, char *content_type, void *content, int content_length) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// + +// Allocate a new cache entry with the passed parameters. +// Insert the entry at the head of the doubly-linked list. +// Store the entry in the hashtable as well, indexed by the entry's `path`. +// Increment the current size of the cache. +// If the cache size is greater than the max size: +// - Remove the cache entry at the tail of the linked list. +// - Remove that same entry from the hashtable, using the entry's `path` and the `hashtable_delete` function. +// - Free the cache entry. +// - Ensure the size counter for the number of entries in the cache is correct. (included in dllist_remove_tail(cache);) + +struct cache_entry *new_cache_entry = alloc_entry(path, content_type, content, content_length); + +dllist_insert_head(cache, new_cache_entry); +hashtable_put(cache->index, path, new_cache_entry); +cache->cur_size++; +if(cache->cur_size > cache->max_size) { + struct cache_entry *oldtail_entry = dllist_remove_tail(cache); + hashtable_delete(cache->index, oldtail_entry->path); // ht, key + free_entry(oldtail_entry); + // cache->cur_size--; + } } /** @@ -132,7 +173,17 @@ void cache_put(struct cache *cache, char *path, char *content_type, void *conten */ struct cache_entry *cache_get(struct cache *cache, char *path) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// + +// Goal: Attempt to find the cache entry pointer by `path` in the hash table. +// hashtable_get(struct hashtable *ht, char *key) +// If not found, return `NULL`. + if (hashtable_get(cache->index, path) == NULL) { + return NULL; + } else { +// Move the cache entry to the head of the doubly-linked list. +// Return the cache entry pointer. + dllist_move_to_head(cache, hashtable_get(cache->index, path)); + return cache->head; + } + } diff --git a/src/file.c b/src/file.c index 5277a92f4..0fb0643b0 100644 --- a/src/file.c +++ b/src/file.c @@ -13,7 +13,7 @@ struct file_data *file_load(char *filename) char *buffer, *p; struct stat buf; int bytes_read, bytes_remaining, total_bytes = 0; - + // unsigned long long int bytes_read, bytes_remaining, total_bytes = 0; // Get the file size if (stat(filename, &buf) == -1) { return NULL; @@ -40,8 +40,10 @@ struct file_data *file_load(char *filename) } // Read in the entire file + // unsigned long long int minus_one = -1; while (bytes_read = fread(p, 1, bytes_remaining, fp), bytes_read != 0 && bytes_remaining > 0) { if (bytes_read == -1) { + // if (bytes_read == minus_one) { free(buffer); return NULL; } diff --git a/src/mime.c b/src/mime.c index 4013b5ca8..4de4d99b5 100644 --- a/src/mime.c +++ b/src/mime.c @@ -27,7 +27,7 @@ char *mime_type_get(char *filename) return DEFAULT_MIME_TYPE; } - ext++; + ext++; // move to character after . in ext names (.html) strlower(ext); diff --git a/src/server.c b/src/server.c index ea43306fc..920abfb16 100644 --- a/src/server.c +++ b/src/server.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "net.h" #include "file.h" @@ -53,11 +54,20 @@ int send_response(int fd, char *header, char *content_type, void *body, int cont const int max_response_size = 262144; char response[max_response_size]; - // Build HTTP response and store it in response - /////////////////// - // IMPLEMENT ME! // - /////////////////// + // Remove Body so it can be sent separately from the header (uses 2 send()) + snprintf(response, max_response_size, + "%s\n" + "Content-Type: %s\n" + "Content-Length: %d\n" + "Connection: close\n" + "\n", + header, content_type, content_length + ); + + printf("%s", response); // send() + + int response_length = strlen(response); // Send it all! int rv = send(fd, response, response_length, 0); @@ -66,6 +76,12 @@ int send_response(int fd, char *header, char *content_type, void *body, int cont perror("send"); } + rv = send(fd, body, content_length, 0); + + if (rv < 0) { + perror("send"); + } + return rv; } @@ -76,16 +92,19 @@ int send_response(int fd, char *header, char *content_type, void *body, int cont void get_d20(int fd) { // Generate a random number between 1 and 20 inclusive - - /////////////////// - // IMPLEMENT ME! // - /////////////////// + const int max_int_size = 16; + char random_str_num[max_int_size]; + srand(time(0)); + unsigned int random_num = 1 + rand()%20; - // Use send_response() to send it back as text/plain data + snprintf(random_str_num, max_int_size, "%d\n", random_num + ); - /////////////////// - // IMPLEMENT ME! // - /////////////////// + // printf("%s", random_str_num); + + int r_str_num_len = strlen(random_str_num); + + send_response(fd, "HTTP/1.1 200 OK", "text/plain", random_str_num, r_str_num_len); } /** @@ -119,9 +138,51 @@ void resp_404(int fd) */ void get_file(int fd, struct cache *cache, char *request_path) { - /////////////////// - // IMPLEMENT ME! // - /////////////////// +// 1. Check if the file is in the cache. +// 2. If it is, serve it. We're done. +// 3. If it's not in the cache, load it from disk. +// 4. Save it in the cache. +// 5. Serve it. + + struct cache_entry * ce = cache_get(cache, request_path); + + if (ce != NULL) { + send_response(fd, "HTTP/1.1 200 OK", ce->content_type, ce->content, ce->content_length); + } + + char filepath[4096]; + // char filepath[8388608]; + struct file_data *filedata; + char *mime_type; + + // Fetch Server Root file + snprintf(filepath, sizeof filepath, "%s/%s", SERVER_ROOT, request_path); + filedata = file_load(filepath); + + + if (filedata == NULL) { + if (strcmp(request_path, "/") == 0) { + request_path = "/index.html"; + snprintf(filepath, sizeof filepath, "%s/%s", SERVER_ROOT, request_path); + filedata = file_load(filepath); + mime_type = mime_type_get(filepath); + send_response(fd, "HTTP/1.1 200 OK", mime_type, filedata->data, filedata->size); + file_free(filedata); + } + resp_404(fd); + return; + } + + mime_type = mime_type_get(filepath); + + if (ce) { + send_response(fd, "HTTP/1.1 200 OK", ce->content_type, ce->content, ce->content_length); + } + + cache_put(cache, request_path, mime_type, filedata->data, filedata->size); + send_response(fd, "HTTP/1.1 200 OK", mime_type, filedata->data, filedata->size); + + file_free(filedata); } /** @@ -135,6 +196,7 @@ char *find_start_of_body(char *header) /////////////////// // IMPLEMENT ME! // (Stretch) /////////////////// + return 0; } /** @@ -142,8 +204,11 @@ char *find_start_of_body(char *header) */ void handle_http_request(int fd, struct cache *cache) { + // const unsigned long long int request_buffer_size = 8388608; const int request_buffer_size = 65536; // 64K char request[request_buffer_size]; + char method[512]; + char path[2048]; // Read request int bytes_recvd = recv(fd, request, request_buffer_size - 1, 0); @@ -154,19 +219,28 @@ void handle_http_request(int fd, struct cache *cache) } - /////////////////// - // IMPLEMENT ME! // - /////////////////// - - // Read the three components of the first request line + sscanf(request, "%s %s", method, path); + printf("Method, path: %s %s\n", method, path); - // If GET, handle the get endpoints + if (strcmp(method, "GET") == 0) { + if(strcmp(path, "/d20") == 0) { + get_d20(fd); + } + else { + get_file(fd, cache, path); + } + } else if (strcmp(method, "POST") == 0) { + send_response(fd, "HTTP/1.1 201 Created", "text/plain", "Stretch ", 8); + } else { + resp_404(fd); + } // Check if it's /d20 and handle that special case // Otherwise serve the requested file by calling get_file() - - // (Stretch) If POST, handle the post request + // (Stretch) If POST, handle the post request + // memset(path, 0, sizeof(path)); + // memset(word, 0, sizeof(word)); } /** @@ -179,7 +253,7 @@ int main(void) char s[INET6_ADDRSTRLEN]; struct cache *cache = cache_create(10, 0); - + // Get a listening socket int listenfd = get_listener_socket(PORT); @@ -193,7 +267,7 @@ int main(void) // This is the main loop that accepts incoming connections and // forks a handler process to take care of it. The main parent // process then goes back to waiting for new connections. - + // resp_404(listenfd); while(1) { socklen_t sin_size = sizeof their_addr; @@ -209,13 +283,11 @@ int main(void) inet_ntop(their_addr.ss_family, get_in_addr((struct sockaddr *)&their_addr), s, sizeof s); - printf("server: got connection from %s\n", s); + printf("\nserver: got connection from %s\n", s); // newfd is a new socket descriptor for the new connection. // listenfd is still listening for new connections. - handle_http_request(newfd, cache); - close(newfd); } diff --git a/src/serverroot/cat.jpg b/src/serverroot/cat.png similarity index 100% rename from src/serverroot/cat.jpg rename to src/serverroot/cat.png diff --git a/src/serverroot/gem.jpg b/src/serverroot/gem.jpg new file mode 100644 index 000000000..bbc5094d3 Binary files /dev/null and b/src/serverroot/gem.jpg differ