Skip to content

Commit d66bb05

Browse files
committed
add mods 18 & 19: pull queues to Pub/Sub
1 parent ca2ced3 commit d66bb05

16 files changed

+516
-35
lines changed

README.md

+54-33
Large diffs are not rendered by default.

mod18-gaepull/.gcloudignore

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# This file specifies files that are *not* uploaded to Google Cloud Platform
2+
# using gcloud. It follows the same syntax as .gitignore, with the addition of
3+
# "#!include" directives (which insert the entries of the given .gitignore-style
4+
# file at that point).
5+
#
6+
# For more information, run:
7+
# $ gcloud topic gcloudignore
8+
#
9+
.gcloudignore
10+
11+
# Source code control files
12+
.git/
13+
.gitignore
14+
.hgignore
15+
.hg/
16+
17+
# README/text files
18+
LICENSE
19+
*.md
20+
21+
# Tests/results (not in .gitignore)
22+
noxfile.py
23+
test_translate.py
24+
pylintrc
25+
pylintrc.test
26+
27+
# most of .gitignore (except `lib`)
28+
#
29+
# Python
30+
*.py[cod]
31+
__pycache__/
32+
/setup.cfg
33+
34+
# C extensions
35+
*.so
36+
37+
# Packages
38+
*.egg
39+
*.egg-info
40+
dist
41+
build
42+
eggs
43+
.eggs
44+
parts
45+
bin
46+
var
47+
sdist
48+
develop-eggs
49+
.installed.cfg
50+
lib64
51+
*.tgz
52+
53+
# Installer logs
54+
pip-log.txt
55+
56+
# Tests/results
57+
.nox/
58+
.pytest_cache/
59+
.cache
60+
.pytype
61+
.coverage
62+
coverage.xml
63+
*sponge_log.xml
64+
system_tests/local_test_setup
65+
66+
# Mac
67+
.DS_Store
68+
69+
# IDEs/editors
70+
*.sw[op]
71+
*~
72+
.vscode
73+
.idea
74+
75+
# Built documentation
76+
docs/_build
77+
docs.metadata
78+
79+
# Virtual environment
80+
env/

mod18-gaepull/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Module 18 - Add usage of App Engine TaskQueue (pull tasks) to NDB Flask sample app
2+
3+
This repo folder is the corresponding Python 2 code to the [Module 18 codelab](http://g.co/codelabs/pae-migrate-gaepull). The tutorial STARTs with the Python 2 code in the [Module 1 repo folder](/mod1-flask) and leads developers through adding usage of pull tasks via App Engine TaskQueue, culminating in the code in this folder.

mod18-gaepull/app.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
runtime: python27
16+
threadsafe: yes
17+
api_version: 1
18+
19+
handlers:
20+
- url: /.*
21+
script: main.app

mod18-gaepull/appengine_config.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from google.appengine.ext import vendor
16+
17+
# Set PATH to your libraries folder.
18+
PATH = 'lib'
19+
# Add libraries installed in the PATH folder.
20+
vendor.add(PATH)

mod18-gaepull/main.py

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from flask import Flask, render_template, request
16+
from google.appengine.api import taskqueue
17+
from google.appengine.ext import ndb
18+
19+
HOUR = 3600
20+
LIMIT = 10
21+
TASKS = 1000
22+
QUEUE = 'pullq'
23+
app = Flask(__name__)
24+
25+
26+
class Visit(ndb.Model):
27+
'Visit entity registers visitor IP address & timestamp'
28+
visitor = ndb.StringProperty()
29+
timestamp = ndb.DateTimeProperty(auto_now_add=True)
30+
31+
def store_visit(remote_addr, user_agent):
32+
'create new Visit in Datastore and queue request to bump visitor count'
33+
Visit(visitor='{}: {}'.format(remote_addr, user_agent)).put()
34+
q = taskqueue.Queue(QUEUE)
35+
q.add(taskqueue.Task(payload=remote_addr, method='PULL'))
36+
37+
def fetch_visits(limit):
38+
'get most recent visits'
39+
return Visit.query().order(-Visit.timestamp).fetch(limit)
40+
41+
42+
class VisitorCount(ndb.Model):
43+
visitor = ndb.StringProperty(repeated=False, required=True)
44+
counter = ndb.IntegerProperty()
45+
46+
def fetch_counts(limit):
47+
'get top visitors'
48+
return VisitorCount.query().order(-VisitorCount.counter).fetch(limit)
49+
50+
51+
@app.route('/log')
52+
def log_visitors():
53+
'worker processes recent visitor counts and updates them in Datastore'
54+
# tally recent visitor counts from queue then delete those tasks
55+
tallies = {}
56+
q = taskqueue.Queue(QUEUE)
57+
tasks = q.lease_tasks(HOUR, TASKS)
58+
for task in tasks:
59+
visitor = task.payload
60+
tallies[visitor] = tallies.get(visitor, 0) + 1
61+
q.delete_tasks(tasks)
62+
63+
# increment those counts in Datastore and return
64+
for visitor in tallies:
65+
counter = VisitorCount.query(VisitorCount.visitor == visitor).get()
66+
if not counter:
67+
counter = VisitorCount(visitor=visitor, counter=0)
68+
counter.put()
69+
counter.counter += tallies[visitor]
70+
counter.put()
71+
return 'DONE (with %d task[s] logging %d visitor[s])\r\n' % (len(tasks), len(tallies))
72+
73+
74+
@app.route('/')
75+
def root():
76+
'main application (GET) handler'
77+
store_visit(request.remote_addr, request.user_agent)
78+
context = {
79+
'limit': LIMIT,
80+
'visits': fetch_visits(LIMIT),
81+
'counts': fetch_counts(LIMIT),
82+
}
83+
return render_template('index.html', **context)

mod18-gaepull/queue.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
queue:
2+
- name: pullq
3+
mode: pull

mod18-gaepull/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
flask

mod18-gaepull/templates/index.html

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<title>VisitMe Example</title>
5+
<body>
6+
7+
<h1>VisitMe example</h1>
8+
9+
<h3>Top {{ limit }} visitors</h3>
10+
<table border=1 cellspacing=0 cellpadding=2>
11+
<tr><th>Visitor</th><th>Visits</th></tr>
12+
{% for count in counts %}
13+
<tr><td>{{ count.visitor|e }}</td><td align="center">{{ count.counter }}</td></tr>
14+
{% endfor %}
15+
</table>
16+
17+
<h3>Last {{ limit }} visits</h3>
18+
<ul>
19+
{% for visit in visits %}
20+
<li>{{ visit.timestamp.ctime() }} from {{ visit.visitor }}</li>
21+
{% endfor %}
22+
</ul>
23+
24+
</body>
25+
</html>

mod19-pubsub/.gcloudignore

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# This file specifies files that are *not* uploaded to Google Cloud Platform
2+
# using gcloud. It follows the same syntax as .gitignore, with the addition of
3+
# "#!include" directives (which insert the entries of the given .gitignore-style
4+
# file at that point).
5+
#
6+
# For more information, run:
7+
# $ gcloud topic gcloudignore
8+
#
9+
.gcloudignore
10+
11+
# Source code control files
12+
.git/
13+
.gitignore
14+
.hgignore
15+
.hg/
16+
17+
# README/text files
18+
LICENSE
19+
*.md
20+
21+
# Tests/results (not in .gitignore)
22+
noxfile.py
23+
test_translate.py
24+
pylintrc
25+
pylintrc.test
26+
27+
# most of .gitignore (except `lib`)
28+
#
29+
# Python
30+
*.py[cod]
31+
__pycache__/
32+
/setup.cfg
33+
34+
# C extensions
35+
*.so
36+
37+
# Packages
38+
*.egg
39+
*.egg-info
40+
dist
41+
build
42+
eggs
43+
.eggs
44+
parts
45+
bin
46+
var
47+
sdist
48+
develop-eggs
49+
.installed.cfg
50+
lib64
51+
*.tgz
52+
53+
# Installer logs
54+
pip-log.txt
55+
56+
# Tests/results
57+
.nox/
58+
.pytest_cache/
59+
.cache
60+
.pytype
61+
.coverage
62+
coverage.xml
63+
*sponge_log.xml
64+
system_tests/local_test_setup
65+
66+
# Mac
67+
.DS_Store
68+
69+
# IDEs/editors
70+
*.sw[op]
71+
*~
72+
.vscode
73+
.idea
74+
75+
# Built documentation
76+
docs/_build
77+
docs.metadata
78+
79+
# Virtual environment
80+
env/

mod19-pubsub/README.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Module 19 - Migrate from App Engine `taskqueue` (pull tasks) to Cloud Pub/Sub (and Python 3)
2+
3+
This repo folder is the corresponding Python 2 code to the [Module 19 codelab](http://g.co/codelabs/pae-migrate-pubsub). The tutorial STARTs with the Python 2 code in the [Module 18 repo folder](/mod18-gaepull) and leads developers through its migration from pull tasks via App Engine `taskqueue` to Cloud Pub/Sub and Python 3, culminating in the code in this folder. The migration from App Engine `ndb` to Cloud NDB, covered in Module 2, also takes place.
4+
5+
**NOTE:** While this solution Python 2 compatible, a library conflict on App Engine servers does not allow it to run properly under Python 2 on App Engine.)

mod19-pubsub/app.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
runtime: python310

0 commit comments

Comments
 (0)