summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSimeon Simeonov2023-01-18 21:12:31 +0100
committerSimeon Simeonov2023-01-18 21:12:31 +0100
commit6dcd1727eb9d5dbfcb8f6e9599ada10d33063bac (patch)
tree57d6aa95763fd57835316cca7ce1ae33f3f19ec2
parenta33898d9db2f0e0e9b3da6f1b329b4045b65af11 (diff)
Remove some old presentations
Diffstat
-rw-r--r--notebooks/python/python_intro.ipynb2
-rw-r--r--notebooks/python/python_oo.ipynb107
-rwxr-xr-xreveal.js/dataaccess.html360
-rw-r--r--reveal.js/dataporten.html118
-rw-r--r--reveal.js/demo.html4
-rw-r--r--reveal.js/evalg.html274
-rwxr-xr-xreveal.js/flow.html194
-rw-r--r--reveal.js/gateway.html152
-rw-r--r--reveal.js/mqprod.html438
-rw-r--r--reveal.js/postgresql_tuning.html233
-rw-r--r--reveal.js/python-oo.html145
-rw-r--r--reveal.js/python.html680
-rw-r--r--reveal.js/rabbitmq.html528
-rw-r--r--[-rwxr-xr-x]reveal.js/sqlalchemy.html285
-rw-r--r--reveal.js/timetravel.html214
15 files changed, 1067 insertions, 2667 deletions
diff --git a/notebooks/python/python_intro.ipynb b/notebooks/python/python_intro.ipynb
index 3c49a14..0168839 100644
--- a/notebooks/python/python_intro.ipynb
+++ b/notebooks/python/python_intro.ipynb
@@ -1253,7 +1253,7 @@
1253 "name": "python", 1253 "name": "python",
1254 "nbconvert_exporter": "python", 1254 "nbconvert_exporter": "python",
1255 "pygments_lexer": "ipython3", 1255 "pygments_lexer": "ipython3",
1256 "version": "3.10.8" 1256 "version": "3.11.1"
1257 } 1257 }
1258 }, 1258 },
1259 "nbformat": 4, 1259 "nbformat": 4,
diff --git a/notebooks/python/python_oo.ipynb b/notebooks/python/python_oo.ipynb
index b56aa26..1c655d0 100644
--- a/notebooks/python/python_oo.ipynb
+++ b/notebooks/python/python_oo.ipynb
@@ -179,7 +179,8 @@
179 "\"The Zen of Python\" (import this) is still relevant and should be followed :)\n", 179 "\"The Zen of Python\" (import this) is still relevant and should be followed :)\n",
180 "\n", 180 "\n",
181 "\"Python will remain a dynamically typed language, and the authors have no desire to ever make type hints mandatory, even by convention.\" - PEP484\n", 181 "\"Python will remain a dynamically typed language, and the authors have no desire to ever make type hints mandatory, even by convention.\" - PEP484\n",
182 "\n" 182 "\n",
183 "Class (type) names should normally use the *CapWords* convention - CapitalizedWords (or *CapWords*, or *CamelCase*). This is also sometimes known as *StudlyCaps*. ASCII characters and English names only!"
183 ] 184 ]
184 }, 185 },
185 { 186 {
@@ -225,17 +226,17 @@
225 "name": "stdout", 226 "name": "stdout",
226 "output_type": "stream", 227 "output_type": "stream",
227 "text": [ 228 "text": [
228 "car1.get_obj_info_str() = \"I am <__main__.Car object at 0x7f5630542810> with id 140008154736656 from <class '__main__.Car'> with id 94904583728688\"\n", 229 "car1.get_obj_info_str() = \"I am <__main__.Car object at 0x7fc7add96f10> with id 140495591927568 from <class '__main__.Car'> with id 94628241379728\"\n",
229 "car2.get_obj_info_str() = \"I am <__main__.Car object at 0x7f5633ec8190> with id 140008215052688 from <class '__main__.Car'> with id 94904583728688\"\n", 230 "car2.get_obj_info_str() = \"I am <__main__.Car object at 0x7fc7aef459d0> with id 140495610468816 from <class '__main__.Car'> with id 94628241379728\"\n",
230 "car1.model = 'BMW', car1.reg_nr = 'EC76183', car1.extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140008154734272, id(car1.__init__) = 140008154735680\n", 231 "car1.model = 'BMW', car1.reg_nr = 'EC76183', car1.extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140495591909184, id(car1.get_obj_info_str) = 140495591927808\n",
231 "car2.model = 'Scoda', car2.reg_nr = 'BD77655', car2.extras = ['GPSnav'], id(car2.cls_extras) = 140008154734272, id(car2.__init__) = 140008154736768\n", 232 "car2.model = 'Scoda', car2.reg_nr = 'BD77655', car2.extras = ['GPSnav'], id(car2.cls_extras) = 140495591909184, id(car2.get_obj_info_str) = 140495591927936\n",
232 "id(Car.cls_extras) = 140008154734272, id(Car.__init__) = 140008154484864\n", 233 "id(Car.cls_extras) = 140495591909184, id(Car.get_obj_info_str) = 140495591682944\n",
233 "car1.cls_extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140008154734272\n", 234 "car1.cls_extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140495591909184\n",
234 "car2.cls_extras = ['GPSnav', 'Sound system'], id(car2.cls_extras) = 140008154734272\n", 235 "car2.cls_extras = ['GPSnav', 'Sound system'], id(car2.cls_extras) = 140495591909184\n",
235 "True\n", 236 "True\n",
236 "hasattr(car1, 'import_tax_paid') = True\n", 237 "hasattr(car1, 'import_tax_paid') = True\n",
237 "hasattr(car2, 'import_tax_paid') = False\n", 238 "hasattr(car2, 'import_tax_paid') = False\n",
238 "id(car1.__class__) = 94904583728688, id(car2.__class__) = 94904583728688, id(Car) = 94904583728688\n" 239 "id(car1.__class__) = 94628241379728, id(car2.__class__) = 94628241379728, id(Car) = 94628241379728\n"
239 ] 240 ]
240 } 241 }
241 ], 242 ],
@@ -244,6 +245,7 @@
244 "\n", 245 "\n",
245 " cls_extras = [] # class attribute - shared by all instances\n", 246 " cls_extras = [] # class attribute - shared by all instances\n",
246 " # cls_extras: list = [] is also possible\n", 247 " # cls_extras: list = [] is also possible\n",
248 " car_count = 0\n",
247 "\n", 249 "\n",
248 " def __init__(self, model: str, reg_nr: str, extras: list):\n", 250 " def __init__(self, model: str, reg_nr: str, extras: list):\n",
249 " \"\"\"\n", 251 " \"\"\"\n",
@@ -262,24 +264,40 @@
262 "\n", 264 "\n",
263 " # self.cls_extras = extras # will create a *NEW instance attribute* called 'cls_extras'\n", 265 " # self.cls_extras = extras # will create a *NEW instance attribute* called 'cls_extras'\n",
264 " \n", 266 " \n",
267 " Car.car_count += 1 # increase Car.car_count by 1 every time a car is created \n",
268 " \n",
269 " def __del__(self):\n",
270 " \"\"\"\n",
271 " Called when the instance is about to be destroyed.\n",
272 "\n",
273 " This is also called a finalizer or (improperly) a destructor.\n",
274 " \"\"\"\n",
275 " Car.car_count -= 1 # decrease by 1 every time a Car object is destroyed\n",
276 "\n",
265 " def get_obj_info_str(self): # regular method that becomes instance attribute\n", 277 " def get_obj_info_str(self): # regular method that becomes instance attribute\n",
266 " return f\"I am {self} with id {id(self)} from {self.__class__} with id {id(self.__class__)}\"\n", 278 " return f\"I am {self} with id {id(self)} from {self.__class__} with id {id(self.__class__)}\"\n",
267 "\n", 279 "\n",
268 "car1 = Car(\"BMW\", \"EC76183\", [\"GPSnav\", \"Sound system\"])\n", 280 "car1 = Car(\"BMW\", \"EC76183\", [\"GPSnav\", \"Sound system\"])\n",
269 "car2 = Car(\"Scoda\", \"BD77655\", [\"GPSnav\"])\n", 281 "car2 = Car(\"Scoda\", \"BD77655\", [\"GPSnav\"])\n",
270 "\n", 282 "\n",
271 "print(f\"{car1.get_obj_info_str() = }\")\n", 283 "print(f\"{car1.get_obj_info_str() = }\") # same as calling Car.get_object_info_str(car1)\n",
272 "print(f\"{car2.get_obj_info_str() = }\")\n", 284 "print(f\"{car2.get_obj_info_str() = }\")\n",
273 "\n", 285 "\n",
274 "# default behavior for `object.attr`:\n", 286 "# default behavior for `object.attr`:\n",
275 "# getter:\n", 287 "# getter:\n",
276 "# - checks if 'attr' is an instance attribute\n", 288 "# - checks if 'attr' is an instance attribute\n",
277 "# - checks if 'attr' is a class attribute\n", 289 "# - checks if 'attr' is a class attribute (through the method resolution order - MRO)\n",
278 "# - raises AttributeError\n", 290 "# - raises AttributeError\n",
279 "# setter: (re)defines an instance attribute\n", 291 "# setter:\n",
280 "print(f\"{car1.model = }, {car1.reg_nr = }, {car1.extras = }, {id(car1.cls_extras) = }, {id(car1.__init__) = }\")\n", 292 "# - (re)defines an instance attribute\n",
281 "print(f\"{car2.model = }, {car2.reg_nr = }, {car2.extras = }, {id(car2.cls_extras) = }, {id(car2.__init__) = }\")\n", 293 "\n",
282 "print(f\"{id(Car.cls_extras) = }, {id(Car.__init__) = }\")\n", 294 "# Details not covered in this course:\n",
295 "# Car.x is translated to Car.__dict__[\"x\"] (...through the MRO)\n",
296 "# car1.x is translated to car1.__dict__[\"x\"] (if not found ... Car.x (see above), if not found car1.__getattr__(\"x\") is called)\n",
297 "\n",
298 "print(f\"{car1.model = }, {car1.reg_nr = }, {car1.extras = }, {id(car1.cls_extras) = }, {id(car1.get_obj_info_str) = }\")\n",
299 "print(f\"{car2.model = }, {car2.reg_nr = }, {car2.extras = }, {id(car2.cls_extras) = }, {id(car2.get_obj_info_str) = }\")\n",
300 "print(f\"{id(Car.cls_extras) = }, {id(Car.get_obj_info_str) = }\")\n",
283 "car1.cls_extras.extend(car1.extras)\n", 301 "car1.cls_extras.extend(car1.extras)\n",
284 "# car1.cls_extras = car1.extras # N.B. This will create a *NEW instance attribute* called 'cls_extras'\n", 302 "# car1.cls_extras = car1.extras # N.B. This will create a *NEW instance attribute* called 'cls_extras'\n",
285 "print(f\"{car1.cls_extras = }, {id(car1.cls_extras) = }\")\n", 303 "print(f\"{car1.cls_extras = }, {id(car1.cls_extras) = }\")\n",
@@ -289,7 +307,7 @@
289 "car1.import_tax_paid = True # 'import_tax_paid' will only be present in car1\n", 307 "car1.import_tax_paid = True # 'import_tax_paid' will only be present in car1\n",
290 "print(car1.import_tax_paid)\n", 308 "print(car1.import_tax_paid)\n",
291 "print(f\"{hasattr(car1, 'import_tax_paid') = }\") # Out: True\n", 309 "print(f\"{hasattr(car1, 'import_tax_paid') = }\") # Out: True\n",
292 "print(f\"{hasattr(car2, 'import_tax_paid') = }\") # Out: False\n", 310 "print(f\"{hasattr(car2, 'import_tax_paid') = }\") # Out: False - no 'import_tax_paid' attribute in object car2\n",
293 "\n", 311 "\n",
294 "# .__class__ instance attribute pointing to its class (type) will be created\n", 312 "# .__class__ instance attribute pointing to its class (type) will be created\n",
295 "print(f\"{id(car1.__class__) = }, {id(car2.__class__) = }, {id(Car) = }\")" 313 "print(f\"{id(car1.__class__) = }, {id(car2.__class__) = }, {id(Car) = }\")"
@@ -705,6 +723,7 @@
705 "name": "stdout", 723 "name": "stdout",
706 "output_type": "stream", 724 "output_type": "stream",
707 "text": [ 725 "text": [
726 "2:2\n",
708 "__mul__ called\n", 727 "__mul__ called\n",
709 "vector * 3 = Vector(start=Point(x=2, y=2), end=Point(x=12, y=18))\n", 728 "vector * 3 = Vector(start=Point(x=2, y=2), end=Point(x=12, y=18))\n",
710 "__rmul__ called\n", 729 "__rmul__ called\n",
@@ -771,6 +790,7 @@
771 " # __rmul__ = __mul__\n", 790 " # __rmul__ = __mul__\n",
772 "\n", 791 "\n",
773 "vector = Vector(Point(2, 2), Point(4, 6))\n", 792 "vector = Vector(Point(2, 2), Point(4, 6))\n",
793 "print(f\"{vector['start']}\")\n",
774 "print(f\"{vector * 3 = }\")\n", 794 "print(f\"{vector * 3 = }\")\n",
775 "print(f\"{3 * vector = }\")" 795 "print(f\"{3 * vector = }\")"
776 ] 796 ]
@@ -863,7 +883,7 @@
863 "id": "d1d17f26-de6e-4510-b0f0-3ae02761ae24", 883 "id": "d1d17f26-de6e-4510-b0f0-3ae02761ae24",
864 "metadata": {}, 884 "metadata": {},
865 "source": [ 885 "source": [
866 "# Example 2\n", 886 "# Example 3\n",
867 "\n", 887 "\n",
868 "We will create a small and incomplete Animal class hierarchy" 888 "We will create a small and incomplete Animal class hierarchy"
869 ] 889 ]
@@ -973,7 +993,7 @@
973 " super().__init__(\n", 993 " super().__init__(\n",
974 " weight=weight, tooth_replacement=True, alive=alive, **kwargs\n", 994 " weight=weight, tooth_replacement=True, alive=alive, **kwargs\n",
975 " )\n", 995 " )\n",
976 " self_conservation_status = conservation_status\n", 996 " self._conservation_status = conservation_status\n",
977 "\n", 997 "\n",
978 "\n", 998 "\n",
979 "class Cow(Mammal):\n", 999 "class Cow(Mammal):\n",
@@ -1104,6 +1124,7 @@
1104 " \"\"\"\n", 1124 " \"\"\"\n",
1105 "\n", 1125 "\n",
1106 "\n", 1126 "\n",
1127 "# not good! The user of JSONMixin should not have to explicitly know about / use DictMixin\n",
1107 "class JSONConvertableTigerShark(TigerShark, DictMixin, JSONMixin):\n", 1128 "class JSONConvertableTigerShark(TigerShark, DictMixin, JSONMixin):\n",
1108 " \"\"\"A tiger shark class that provides JSON conversion functionality\"\"\"\n", 1129 " \"\"\"A tiger shark class that provides JSON conversion functionality\"\"\"\n",
1109 "\n", 1130 "\n",
@@ -1143,14 +1164,7 @@
1143 "MRO is monotonic when the following is true: if C1 precedes C2 in the linearization of C, then C1 precedes C2 in the linearization of any subclass of C.\n", 1164 "MRO is monotonic when the following is true: if C1 precedes C2 in the linearization of C, then C1 precedes C2 in the linearization of any subclass of C.\n",
1144 "\n", 1165 "\n",
1145 "Not all classes admit a linearization. There are cases, in complicated hierarchies, where it is not possible to derive a class such that its linearization respects all the desired properties. \n", 1166 "Not all classes admit a linearization. There are cases, in complicated hierarchies, where it is not possible to derive a class such that its linearization respects all the desired properties. \n",
1146 "`TypeError: Cannot create a consistent method resolution order (MRO) for bases X, Y` is raised then.\n", 1167 "`TypeError: Cannot create a consistent method resolution order (MRO) for bases X, Y` is raised then."
1147 "\n",
1148 "\n",
1149 "## The Liskov substitution principle (LSP):\n",
1150 "\n",
1151 "Functions that use references to base class objects must be able to use objects of derived classes without knowing it.\n",
1152 "\n",
1153 "In other words: an object (such as a class) may be replaced by a sub-object (such as a class that extends the first class) without breaking the program.\n"
1154 ] 1168 ]
1155 }, 1169 },
1156 { 1170 {
@@ -1233,6 +1247,14 @@
1233 ] 1247 ]
1234 }, 1248 },
1235 { 1249 {
1250 "cell_type": "markdown",
1251 "id": "875e7ffd-4feb-4ef0-b912-0a939c6c2dd1",
1252 "metadata": {},
1253 "source": [
1254 "... **N.B.** The most important thing to note here is that if `super()` is used in `Grandfather1` for this MRO (f.i. object of type `Simeon`) , none of its parents or siblings is called, but `Father` that was probably \"not even born\" when `Grandfather1` was created. :)"
1255 ]
1256 },
1257 {
1236 "cell_type": "code", 1258 "cell_type": "code",
1237 "execution_count": 12, 1259 "execution_count": 12,
1238 "id": "95a1d0cc-0006-4ddc-b08d-d0b8b02acfba", 1260 "id": "95a1d0cc-0006-4ddc-b08d-d0b8b02acfba",
@@ -1274,7 +1296,6 @@
1274 " \"\"\"Produces JSON content for production plans\"\"\"\n", 1296 " \"\"\"Produces JSON content for production plans\"\"\"\n",
1275 " \n", 1297 " \n",
1276 " def to_json(self, **kwargs):\n", 1298 " def to_json(self, **kwargs):\n",
1277 " # super().get_data(**kwargs)\n",
1278 " self.get_data(**kwargs)\n", 1299 " self.get_data(**kwargs)\n",
1279 " prod_plans = self.extract_production_plans(**kwargs)\n", 1300 " prod_plans = self.extract_production_plans(**kwargs)\n",
1280 " return json.dumps(self.__do_some_magic(prod_plans), indent=kwargs.get(\"indent\", 2))\n", 1301 " return json.dumps(self.__do_some_magic(prod_plans), indent=kwargs.get(\"indent\", 2))\n",
@@ -1295,7 +1316,10 @@
1295 "\n", 1316 "\n",
1296 "\n", 1317 "\n",
1297 "cool_ts_proprocessor = CoolTSPreprocessor()\n", 1318 "cool_ts_proprocessor = CoolTSPreprocessor()\n",
1298 "print(cool_ts_proprocessor.to_json(indent=4))" 1319 "print(cool_ts_proprocessor.to_json(indent=4))\n",
1320 "\n",
1321 "# but we want to test CoolTSPreprocessor without the annoying ConnectionManager.get_data method being used\n",
1322 "# simply (temporary or canditionally) replacing ConnectionManager with MockConnectionManager will go against the O in SOLID"
1299 ] 1323 ]
1300 }, 1324 },
1301 { 1325 {
@@ -1311,6 +1335,7 @@
1311 "Using mock data\n", 1335 "Using mock data\n",
1312 "Extracting production plans from data\n", 1336 "Extracting production plans from data\n",
1313 "some magic is happening here...\n", 1337 "some magic is happening here...\n",
1338 "{}\n",
1314 "(<class '__main__.MockedButStillCoolTSPreprocessor'>, <class '__main__.CoolTSPreprocessor'>, <class '__main__.MockConnectionManager'>, <class '__main__.ConnectionManager'>, <class 'object'>)\n" 1339 "(<class '__main__.MockedButStillCoolTSPreprocessor'>, <class '__main__.CoolTSPreprocessor'>, <class '__main__.MockConnectionManager'>, <class '__main__.ConnectionManager'>, <class 'object'>)\n"
1315 ] 1340 ]
1316 } 1341 }
@@ -1320,12 +1345,36 @@
1320 " pass\n", 1345 " pass\n",
1321 "\n", 1346 "\n",
1322 "mocked_but_still_cool_ts_preprocessor = MockedButStillCoolTSPreprocessor()\n", 1347 "mocked_but_still_cool_ts_preprocessor = MockedButStillCoolTSPreprocessor()\n",
1323 "mocked_but_still_cool_ts_preprocessor.to_json(indent=4)\n", 1348 "print(mocked_but_still_cool_ts_preprocessor.to_json(indent=4))\n",
1324 "print(MockedButStillCoolTSPreprocessor.__mro__)" 1349 "print(MockedButStillCoolTSPreprocessor.__mro__)"
1325 ] 1350 ]
1326 }, 1351 },
1327 { 1352 {
1328 "cell_type": "markdown", 1353 "cell_type": "markdown",
1354 "id": "738b9084-27b9-4837-a927-e2087267d483",
1355 "metadata": {},
1356 "source": [
1357 "# Quick tips and final words\n",
1358 "\n",
1359 "- Avoid the classical pitfall: \"I am solving simple problems by simple means (the way I am used to) and could probably use OO when solving more complex problems\"! Solving simple problems is the best way to get used to the OO way of thinking / designing code.\n",
1360 "\n",
1361 "- Take some time to analyze and design your solution and discuss it with other team members and more experienced programmers. Do not rely only on QA (merge requests)!\n",
1362 "\n",
1363 "- Learn to think in terms of objects, like in real world objects (f.i. Car, Animal, TimeSeries, ProductionPlan, DataSource...)!\n",
1364 "\n",
1365 "- Learn to think in terms of abstract classes (types): Classes that will only serve as a foundation to their child classes, but will themselves never be directly instanciated!\n",
1366 "\n",
1367 "- Do not create \"thin\" classes / objects that simply containin some relevant attributes, protected by corresponding properties! The entire functionality (business logic) of the object should be encapsulated there.\n",
1368 "\n",
1369 "- Always make sure that you are not violating some of the SOLID principles (and others described above)! Run \"mental tests\" to test your current design!\n",
1370 "\n",
1371 "- Consider composition (\"has-a\" relationship) before considering inhiritance (\"is-a\" relationship)!\n",
1372 "\n",
1373 "- Read and learn from free software projects written in Python (f.i. on github.com)"
1374 ]
1375 },
1376 {
1377 "cell_type": "markdown",
1329 "id": "540de317-ba77-4f5c-97ba-f1c0bec071fb", 1378 "id": "540de317-ba77-4f5c-97ba-f1c0bec071fb",
1330 "metadata": {}, 1379 "metadata": {},
1331 "source": [ 1380 "source": [
diff --git a/reveal.js/dataaccess.html b/reveal.js/dataaccess.html
deleted file mode 100755
index 4bafccd..0000000
--- a/reveal.js/dataaccess.html
+++ /dev/null
@@ -1,360 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>SQLAlchemy &amp; Odin Data Access</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>SQLAlchemy &amp; Odin Data Access</h2>
28 <h4>Fifty Data Science</h4>
29 </br>
30 <p><small>Simeon Simeonov</small></p>
31 </section>
32
33 <section>
34
35 <section id="fragments">
36 <h2>Agenda</h2>
37 </br>
38 <ul>
39 <span class="fragment"><li>SQLAlchemy - Design & overview</li></span>
40 <span class="fragment"><li>SQLAlchemy - A small practical example</li></span>
41 <span class="fragment"><li>Odin Data Access - Overview and examples</li></span>
42 <span class="fragment"><li>Q &amp; A</li></span>
43 </ul>
44 </section>
45
46 </section>
47
48 <section>
49 <h2>What is SQLAlchemy?</h2>
50 </br>
51 <p><em>SQLAlchemy</em> is a Python library created by Mike Bayer to provide a high-level Pythonic interface to RDBMS such as <em>PostgreSQL</em>, <em>SQLite</em>, <em>MySQL</em>, <em>Oracle</em>, <em>DB2</em>.</p>
52 <p><em>SQLAlchemy</em> includes RDBMS-independent SQL expression language and an <em>object-relational mapper (ORM)</em>.</p>
53 </section>
54
55 <section>
56 <h2>Why use SQLAlchemy?</h2>
57 </br>
58 <ul>
59 <li><p>portability - the programming interface is independent of the type of RDBMS and connector used</p></li>
60 <li><p>security - no more SQL injections</p></li>
61 <li><p>abstraction - no need to bother with complex JOINs</p></li>
62 <li><p>object-orientation - you work with objects and NOT tables and rows</p></li>
63 <li><p>performance - exploits the likehood of reusing a particular query</p></li>
64 <li><p>flexibility - you can override almost anything</p></li>
65 </ul>
66 </section>
67
68 <section>
69 <h2>Basic architecture</h2>
70 <p><em>SQLAlchemy</em> consists of several components, including the the <em>ORM</em>.</p>
71 <ul>
72 <li><em>Engine</em>- manages the connection pool and the RDBMS-independent SQL dialect layer</li>
73 <li><em>MetaData</em> - used to collect and organize information about your table layout (schema)</li>
74 <li><em>Session</em> - establishes all conversations with the RDBMS and represents a "holding zone" for all the objects which you've loaded or associated with it during its lifespan</li>
75 <li><em>SQL expression language</em> - provides an API to execute your queries and updates against your tables, all from Python, and all in a database-independent way (low-level interface)</li>
76 <li><em>ORM</em> - provides a convenient way to add database persistence to your Python objects withoutrequiring you to design your objects around the database, or the database around the objects (high-level interface)</li>
77 </ul>
78 <img src="images/sqlalchemy/sqla_arch.png"></img>
79 </section>
80
81 <section data-auto-animate>
82 <h2>Example</h2>
83 <p><em>SQLAlchemy</em> gives us the choice between <em>classical mapping</em> and the newer <em>declarative mapping</em></p>
84 <pre data-id="code-animation">
85 <code class="python" data-trim data-line-numbers type="text/template">
86 import sqlalchemy
87 sqlalchemy.__version__
88
89 from sqlalchemy import create_engine
90
91 # engine = create_engine("postgresql+psycopg2://user:zipassword@localhost/mydb" , echo=True)
92 # The string form of the URL is dialect+driver://user:password@host/dbname[?key=value..],
93 # where dialect is a database name such as mysql, oracle, postgresql, etc.,
94 # and driver the name of a DBAPI, such as psycopg2, pyodbc, cx_oracle
95 # The echo flag is a shortcut to setting up SQLAlchemy logging,
96 # which is accomplished via Python’s standard logging module.
97 # With it enabled, we’ll see all the generated SQL produced.
98 # engine = create_engine("sqlite:///library.db", echo=True)
99 engine = create_engine("sqlite:///:memory:", echo=True)
100
101 metadata = MetaData()
102
103 from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey
104
105 authors_table = Table(
106 "authors",
107 metadata,
108 Column("id", Integer, primary_key=True),
109 Column("name", String),
110 ) # Column("name", String(50)) is possible
111
112 books_table = Table(
113 "books",
114 metadata,
115 Column("id", Integer, primary_key=True),
116 Column("title", String),
117 Column("description", String),
118 Column("author_id", ForeignKey('authors.id')),
119 )
120
121 metadata.create_all(engine) # creates the tables
122 </code>
123 </pre>
124 </section>
125
126 <section data-auto-animate>
127 <h2>Example (cont...)</h2>
128 <h4 data-id="code-title">Use of <em>SQL expression language</em></h4>
129 <pre data-id="code-animation">
130 <code class="python" data-trim data-line-numbers type="text/template">
131 insert_stmt = authors_table.insert(bind=engine)
132 type(insert_stmt)
133 # Out: &lt;class 'sqlalchemy.sql.expression.Insert'&gt;
134 print(insert_stmt)
135 # Out: INSERT INTO authors (id, name) VALUES (:id,:name)
136
137 compiled_stmt = insert_stmt.compile()
138 print(compiled_stmt.params)
139 # Out: {'id': None, 'name': None}
140
141 insert_stmt.execute(name="Alexandre Dumas") # insert a single entry
142 insert_stmt.execute([{"name": 'Mr X'},{'name': 'Mr Y'}]) # a list of entries
143
144 metadata.bind = engine # no need to explicitly bind the engine from now on
145 select_stmt = authors_table.select(authors_table.c.id==2)
146 result = select_stmt.execute()
147 result.fetchall()
148 # Out: [(1, u'Mr X')]
149
150 del_stmt = authors_table.delete()
151 del_stmt.execute(whereclause=text("name='Mr Y'"))
152 del_stmt.execute() # delete all
153 </code>
154 </pre>
155 </section>
156
157 <section data-auto-animate>
158 <h2>Example (cont...)</h2>
159 <h4 data-id="code-title">Use of <em>classical mapping</em></h4>
160 <pre data-id="code-animation">
161 <code class="python" data-trim data-line-numbers type="text/template">
162 from sqlalchemy.orm import mapper
163 from sqlalchemy.orm import relationship, backref
164
165 class Author:
166 def __init__(self, name):
167 self.name = name
168
169 def __str__(self):
170 return self.name
171
172
173 class Book:
174 def __init__(self, title, description, author):
175 self.title = title
176 self.description = description
177 self.author = author
178
179 def __str__(self):
180 return self.title
181
182 mapper(Book, books_table)
183 mapper(Author, authors_table, properties = {"books": relation(Book, backref="author")})
184 </code>
185 </pre>
186 </section>
187
188 <section data-auto-animate>
189 <h2>Example (cont...)</h2>
190 <h4 data-id="code-title">Doing the same thing the easy way with <em>declarative mapping</em></h4>
191 <pre data-id="code-animation">
192 <code class="python" data-trim data-line-numbers type="text/template">
193 from sqlalchemy.ext.declarative import declarative_base
194 from sqlalchemy.orm import relationship, backref
195
196 Base = declarative_base()
197
198 class Author(Base):
199 __tablename__ = "authors"
200
201 id = Column(Integer, primary_key=True)
202 name = Column(String)
203
204 def __init__(self, name):
205 self.name = name
206
207 def __str__(self):
208 return self.name
209
210
211 class Book(Base):
212 __tablename__ = "books" # self.__table__ will be available for our objects
213
214 id = Column(Integer, primary_key=True)
215 title = Column(String)
216 description = Column(String)
217 author_id = Column(Integer, ForeignKey("authors.id"))
218 author = relationship(Author, backref=backref("books", order_by=title))
219
220 def __init__(self, title, description, author):
221 self.title = title
222 self.description = description
223 self.author = author
224
225 def __str__(self):
226 return self.title
227
228 Base.metadata.create_all(engine)# create tables
229 </code>
230 </pre>
231 </section>
232
233 <section data-auto-animate>
234 <h2>Example (cont...)</h2>
235 <h4 data-id="code-title">Creating instances</h4>
236 <pre data-id="code-animation">
237 <code class="python" data-trim data-line-numbers type="text/template">
238 from sqlalchemy.orm import sessionmaker
239
240 Session = sessionmaker(bind=engine) # bound session
241 session = Session()
242
243 author_1 = Author("Richard Dawkins")
244 author_2 = Author("Matt Ridley")
245
246 book_1 = Book("The Red Queen", "A popular science book", author_2)
247 book_2 = Book("The Selfish Gene", "A popular science book", author_1)
248 book_3 = Book("The Blind Watchmaker", "The theory of evolutio", author_1) # typo
249
250 session.add(author_1)
251 session.add(author_2)
252 session.add(book_1)
253 session.add(book_2)
254 session.add(book_3) # or simply session.add_all([author_1, author_2, book_1, book_2, book_3])
255
256 # session.flush()
257 session.commit() # flushes (issues the statements and sends them to the RDBMS) and commits
258
259 book_3.description = "The theory of evolution" # update the object
260 book_3 in session # check whether the object is in the session
261 # Out: True
262
263 session.commit()
264 </code>
265 </pre>
266 </section>
267
268 <section data-auto-animate>
269 <h2>Example (cont...)</h2>
270 <h4 data-id="code-title">Queries</h4>
271 <pre data-id="code-animation">
272 <code class="python" data-trim data-line-numbers type="text/template">
273 session.query(Book).order_by(Book.id) # returns a Query instance with a .statement attribute
274 session.query(Book).order_by(Book.id).all() # returns an object-list
275
276 # return all book objects where title == "The Selfish Gene"
277 session.query(Book).filter(Book.title == "The Selfish Gene").order_by(Book.id).all()
278
279 # using LIKE
280 session.query(Book).filter(Book.title.like("The%")).order_by(Book.id).all()
281
282 query = session.query(Book).filter(Book.id == 9).order_by(Book.id)
283 query.count() # returns 0L
284 query.all() # returns an empty list
285 query.first() # returns None
286 query.one() # raises NoResultFound exception
287
288 query = session.query(Book).filter(Book.id == 1).order_by(Book.id)
289 book_1 = query.one()
290 book_1.description # returns "A popular science book"
291 book_1.author.books # returns a list of Book-objects representing all the books from the same author.
292
293 # get a list of all Book-instances where the author"s name is "Richard Dawkins"
294 session.query(Book).filter(Book.author_id == Author.id).filter(Author.name == "Richard Dawkins").all()
295 session.query(Book).join(Author).filter(Author.name == "Richard Dawkins").all()
296 session.query(Book).\
297 from_statement("SELECT b.* FROM books b, authors a WHERE b.author_id = a.id AND a.name=:name").\
298 params(name="Richard Dawkins").all()
299 </code>
300 </pre>
301 </section>
302
303 <section data-background="http://i.giphy.com/90F8aUepslB84.gif">
304 <h2>... simply amazing!</h2>
305 </section>
306
307 <section>
308 <h2>Odin Data Access</h2>
309 <p><a href="https://gitlab.fifty.eu/odin/data-science/odin-data-access">https://gitlab.fifty.eu/odin/data-science/odin-data-access</a></p>
310 <p>Provides the <em>odin_data_access</em> module with some of the following entities:</p>
311 <ul>
312 <li><em>session</em> module for creating Session objects in a Fifty environment via <em>get_session</em> and <em>get_session_from_env</em></li>
313 <li><em>DBBase</em> - a tiny base-class that can be used as a mixin for mapping</li>
314 <li><em>DBTableAccessor</em> - tiny wrapper for <em>Session</em> and <em>Table</em> as well a "facade" for <em>Query</em> and <em>pandas</em></li>
315 </ul>
316 <p>The <em>DBTableAccessor</em>:</p>
317 <ul>
318 <li>represents a single RDBMS-table that is introspected during instance initialization (<em>__init__</em>)</li>
319 <li>uses a mix of <em>ORM</em> and <em>SQL expression language</em> functionality</li>
320 <li>contins a mapper for mapping custom classes to its corresponding <em>Table</em></li>
321 <span class="fragment"><li>used by Batman and Chuck Norris (source: incomplete Google search)</li></span>
322 </ul>
323 </section>
324
325 <section>
326 <h1>Demo</h1>
327 <p>Now watch this well-rehearsed demo...</p>
328 </section>
329
330 <section>
331 <h1>Q &amp; A</h1>
332 </section>
333
334 </div>
335 </div>
336
337 <script src="dist/reveal.js"></script>
338 <script src="plugin/zoom/zoom.js"></script>
339 <script src="plugin/notes/notes.js"></script>
340 <script src="plugin/search/search.js"></script>
341 <script src="plugin/markdown/markdown.js"></script>
342 <script src="plugin/highlight/highlight.js"></script>
343 <script>
344
345 // Also available as an ES module, see:
346 // https://revealjs.netlify.app/initialization/
347 Reveal.initialize({
348 controls: true,
349 progress: true,
350 center: true,
351 hash: true,
352
353 // Learn about plugins: https://revealjs.netlify.app/plugins/
354 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
355 });
356
357 </script>
358
359 </body>
360</html>
diff --git a/reveal.js/dataporten.html b/reveal.js/dataporten.html
deleted file mode 100644
index 6c9f941..0000000
--- a/reveal.js/dataporten.html
+++ /dev/null
@@ -1,118 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>Utviklerforum: Dataporten &amp; eValg 3.0</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9
10 <meta name="viewport" content="width=device-width, initial-scale=1.0">
11
12 <link rel="stylesheet" href="dist/reset.css">
13 <link rel="stylesheet" href="dist/reveal.css">
14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 </head>
19 <body>
20 <div class="reveal">
21
22 <!-- Any section element inside of this container is displayed as a slide -->
23 <div class="slides">
24 <section>
25 <h1>Dataporten &amp; eValg 3.0</h1>
26 <h3>Utviklerforum (31.08.2017)</h3>
27 <br/>
28 <p>
29 <small>Simeon Simeonov</small>
30 </p>
31 </section>
32
33 <section>
34 <h2>Agenda</h2>
35 <br/>
36 <ul>
37 <li><p>Intro</p></li>
38 <li><p>The big picture</p></li>
39 <li><p>Flow</p></li>
40 <li><p>Q &amp; A</p></li>
41 </ul>
42 </section>
43
44 <section>
45 <section>
46 <h2>Introduction</h2>
47 <p><em>Evalg 3.0</em></p>
48 <ul>
49 <li><p>Development started in 2017</p></li>
50 <li><p>Dataporten / Feide (OIDC with discovery) ... perhaps others in the future?</p></li>
51 <li><p>Python 3.6 / Flask / React / pyoic / pyJWT ...</p></li>
52 <li><p>Using fnr, feide-id, names...</p></li>
53 </ul>
54 </section>
55 <section>
56 <h3>Challenges with Dataporten</h3>
57 <ul>
58 <li><p>React</p></li>
59 <li><p>Flexibility (using other providers (OP))</p></li>
60 <li><p>Testing (users, firewall(s))</p></li>
61 </ul>
62 </section>
63 </section>
64
65 <section>
66 <h2>The big picture</h2>
67 <img src="images/dataporten/dataporten_the_big_picture1.png"></img>
68 </section>
69
70 <section>
71 <section>
72 <h2>Flow</h2>
73 <img src="images/dataporten/flow1.png"></img>
74 </section>
75 <section>
76 <h3>Challenges</h3>
77 <ul>
78 <li><p>Not distributing client secrets</p></li>
79 <li><p>Checking state and verifying signatures</p></li>
80 <li><p>Flexibility (using other providers (OP))</p></li>
81 <li><p>Testing (users, firewall(s))</p></li>
82 </ul>
83 </section>
84 </section>
85
86 <section>
87 <h2>Q &amp; A</h2>
88 <h3>???</h3>
89 <p><em>“Judge a man by his questions rather than by his answers.”</em> - Voltaire</p>
90 </section>
91
92 </div>
93 </div>
94
95 <script src="dist/reveal.js"></script>
96 <script src="plugin/zoom/zoom.js"></script>
97 <script src="plugin/notes/notes.js"></script>
98 <script src="plugin/search/search.js"></script>
99 <script src="plugin/markdown/markdown.js"></script>
100 <script src="plugin/highlight/highlight.js"></script>
101 <script>
102
103 // Also available as an ES module, see:
104 // https://revealjs.netlify.app/initialization/
105 Reveal.initialize({
106 controls: true,
107 progress: true,
108 center: true,
109 hash: true,
110
111 // Learn about plugins: https://revealjs.netlify.app/plugins/
112 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
113 });
114
115 </script>
116
117 </body>
118</html>
diff --git a/reveal.js/demo.html b/reveal.js/demo.html
index ab11f77..39b014d 100644
--- a/reveal.js/demo.html
+++ b/reveal.js/demo.html
@@ -86,7 +86,7 @@
86 86
87 <section data-auto-animate> 87 <section data-auto-animate>
88 <h2 data-id="code-title">Pretty Code</h2> 88 <h2 data-id="code-title">Pretty Code</h2>
89 <pre data-id="code-animation"><code class="hljs" data-trim data-line-numbers> 89 <pre data-id="code-animation"><code class="hljs javascript" data-trim data-line-numbers>
90 import React, { useState } from 'react'; 90 import React, { useState } from 'react';
91 91
92 function Example() { 92 function Example() {
@@ -102,7 +102,7 @@
102 102
103 <section data-auto-animate> 103 <section data-auto-animate>
104 <h2 data-id="code-title">With animations</h2> 104 <h2 data-id="code-title">With animations</h2>
105 <pre data-id="code-animation"><code class="hljs" data-trim data-line-numbers="|4,8-11|17|22-24"><script type="text/template"> 105 <pre data-id="code-animation"><code class="hljs javascript" data-trim data-line-numbers="|4,8-11|17|22-24"><script type="text/template">
106 import React, { useState } from 'react'; 106 import React, { useState } from 'react';
107 107
108 function Example() { 108 function Example() {
diff --git a/reveal.js/evalg.html b/reveal.js/evalg.html
index e163950..d68a345 100644
--- a/reveal.js/evalg.html
+++ b/reveal.js/evalg.html
@@ -1,160 +1,162 @@
1<!doctype html> 1<!doctype html>
2<html lang="en"> 2<html lang="en">
3 <head> 3 <head>
4 <meta charset="utf-8"> 4 <meta charset="utf-8">
5 <title>Utviklerforum: eValg 3</title> 5 <title>Utviklerforum: eValg 3</title>
6 <meta name="author" content="Simeon Simeonov"> 6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes"> 7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"> 8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> 9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 <link rel="stylesheet" href="css/reveal.css">
11 <link rel="stylesheet" href="css/theme/serif.css" id="theme">
12 10
13 <!-- Theme used for syntax highlighting of code --> 11 <link rel="stylesheet" href="dist/reset.css">
14 <link rel="stylesheet" href="lib/css/zenburn.css"> 12 <link rel="stylesheet" href="dist/reveal.css">
15 </head>
16 <body>
17 <div class="reveal">
18 13
19 <!-- Any section element inside of this container is displayed as a slide --> 14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
20 <div class="slides">
21 <section>
22 <h1>eValg 3</h1>
23 <h3>Utviklerforum (28.11.2019)</h3>
24 <br/>
25 <p>Simeon Simeonov (<em>sgs</em> @ USITINT)</p>
26 </section>
27 15
28 <section> 16 <!-- Theme used for syntax highlighting of code -->
29 <h2>Agenda</h2> 17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
30 <br/> 18 </head>
31 <ul> 19 <body>
32 <li><p>Introduction</p></li> 20 <div class="reveal">
33 <li><p>Components</p></li>
34 <li><p>Development</p></li>
35 <li><p>Q &amp; A</p></li>
36 </ul>
37 </section>
38 21
39 <section> 22 <!-- Any section element inside of this container is displayed as a slide -->
40 <h2>Introduction</h2> 23 <div class="slides">
41 <p><em>eValg 3</em></p> 24 <section>
42 <ul> 25 <h1>eValg 3</h1>
43 <li><p>Development started in 2017 and then again in 2018</p></li> 26 <h3>Utviklerforum (28.11.2019)</h3>
44 <li><p>The last trial election (https://valg-pilot.uio.no) ended 15.11.2019</p></li> 27 <br/>
45 <li><p>Currently the eValg team consists of two developers (<em>skh</em>, <em>sgs</em>)</p></li> 28 <p>Simeon Simeonov (<em>sgs</em> @ USITINT)</p>
46 </ul> 29 </section>
47 </section>
48 30
31 <section>
32 <h2>Agenda</h2>
33 <br/>
34 <ul>
35 <li><p>Introduction</p></li>
36 <li><p>Components</p></li>
37 <li><p>Development</p></li>
38 <li><p>Q &amp; A</p></li>
39 </ul>
40 </section>
49 41
50 <section> 42 <section>
51 <section> 43 <h2>Introduction</h2>
52 <h2>Components</h2> 44 <p><em>eValg 3</em></p>
53 <br/> 45 <ul>
54 <ul> 46 <li><p>Development started in 2017 and then again in 2018</p></li>
55 <li><p>Backend</p></li> 47 <li><p>The last trial election (https://valg-pilot.uio.no) ended 15.11.2019</p></li>
56 <li><p>Dataporten</p></li> 48 <li><p>Currently the eValg team consists of two developers (<em>skh</em>, <em>sgs</em>)</p></li>
57 <li><p>Frontend</p></li> 49 </ul>
58 </ul> 50 </section>
59 </section>
60 51
61 <section>
62 <h3>Backend</h3>
63 <ul>
64 <li><p>General: Python >= 3.6, Flask, SQLAlchemy</p></li>
65 <li><p>DB: Alembic, SQLAlchemy-continuum, Flask-migrate</p></li>
66 <li><p>Signing & encryption: pynacl</p></li>
67 <li><p>Querying (GraphQL): graphene, graphene-sqlalchemy</p></li>
68 <li><p>Others: pytz, sentry_sdk, python-json-logger and many more</p></li>
69 </ul>
70 </section>
71 52
72 <section> 53 <section>
73 <h3>Dataporten</h3> 54 <section>
74 <ul> 55 <h2>Components</h2>
75 <li><p>OpenID Connect: Moved from Authorization code flow to Implicit flow (id_token token) in 2018</p></li> 56 <br/>
76 </ul> 57 <ul>
77 </section> 58 <li><p>Backend</p></li>
59 <li><p>Dataporten</p></li>
60 <li><p>Frontend</p></li>
61 </ul>
62 </section>
78 63
79 <section> 64 <section>
80 <h3>Frontend</h3> 65 <h3>Backend</h3>
81 <ul> 66 <ul>
82 <li><p>General: React, react-oidc, GraphQL</p></li> 67 <li><p>General: Python >= 3.6, Flask, SQLAlchemy</p></li>
83 <li><p>Encryption: TweetNaCl - a crypto library in 100 tweets</p></li> 68 <li><p>DB: Alembic, SQLAlchemy-continuum, Flask-migrate</p></li>
84 <li><p>Many, many more libraries and packages</p></li> 69 <li><p>Signing & encryption: pynacl</p></li>
85 </ul> 70 <li><p>Querying (GraphQL): graphene, graphene-sqlalchemy</p></li>
86 </section> 71 <li><p>Others: pytz, sentry_sdk, python-json-logger and many more</p></li>
72 </ul>
73 </section>
87 74
88 </section> 75 <section>
76 <h3>Dataporten</h3>
77 <ul>
78 <li><p>OpenID Connect: Moved from Authorization code flow to Implicit flow (id_token token) in 2018</p></li>
79 </ul>
80 </section>
89 81
82 <section>
83 <h3>Frontend</h3>
84 <ul>
85 <li><p>General: React, react-oidc, GraphQL</p></li>
86 <li><p>Encryption: TweetNaCl - a crypto library in 100 tweets</p></li>
87 <li><p>Many, many more libraries and packages</p></li>
88 </ul>
89 </section>
90 90
91 <section> 91 </section>
92 92
93 <section>
94 <h2>Development</h2>
95 <br/>
96 <ul>
97 <li><p>Improvements</p></li>
98 <li><p>Goals &amp; challenges</p></li>
99 </ul>
100 </section>
101 93
102 <section> 94 <section>
103 <h3>Improvements</h3>
104 <ul>
105 <li><p>Testing (<em>pytest</em>) & CI (<em>Jenkins</em>) - the best in USITINT so far?!? :)</p></li>
106 <li><p>No more FNR* and cross domain access. #GDPR</p></li>
107 <li><p>No more external programs (<em>GnuPG</em>, <em>evalg.jar</em>) and envelopes ...</p></li>
108 <li><p>Insight in all possible election paths in case of drawing</p></li>
109
110 </ul>
111 <p><small><em>* - work in progress</em></small></p>
112 </section>
113 95
114 <section> 96 <section>
115 <h3>Goals &amp; challenges</h3> 97 <h2>Development</h2>
116 <ul> 98 <br/>
117 <li><p>Anyone should be able to create an election</p></li> 99 <ul>
118 <li><p>No private key shall ever live in the backend</p></li> 100 <li><p>Improvements</p></li>
119 <li><p>Log &amp; audit all changes</p></li> 101 <li><p>Goals &amp; challenges</p></li>
120 <li><p>It is never too late to change a pollbook</p></li> 102 </ul>
121 <li><p>It is never too late to master public-key cryptography ... :)</p></li> 103 </section>
122 </ul>
123 </section>
124 104
125 </section> 105 <section>
106 <h3>Improvements</h3>
107 <ul>
108 <li><p>Testing (<em>pytest</em>) & CI (<em>Jenkins</em>) - the best in USITINT so far?!? :)</p></li>
109 <li><p>No more FNR* and cross domain access. #GDPR</p></li>
110 <li><p>No more external programs (<em>GnuPG</em>, <em>evalg.jar</em>) and envelopes ...</p></li>
111 <li><p>Insight in all possible election paths in case of drawing</p></li>
112
113 </ul>
114 <p><small><em>* - work in progress</em></small></p>
115 </section>
126 116
117 <section>
118 <h3>Goals &amp; challenges</h3>
119 <ul>
120 <li><p>Anyone should be able to create an election</p></li>
121 <li><p>No private key shall ever live in the backend</p></li>
122 <li><p>Log &amp; audit all changes</p></li>
123 <li><p>It is never too late to change a pollbook</p></li>
124 <li><p>It is never too late to master public-key cryptography ... :)</p></li>
125 </ul>
126 </section>
127 127
128 <section> 128 </section>
129 <h2>Q &amp; A</h2>
130 <h3>???</h3>
131 <p><em>“Old questions are not answered—they only go out of fashion”</em> - Donald Schön</p>
132 </section>
133 </div>
134 </div>
135 129
136 <script src="lib/js/head.min.js"></script>
137 <script src="js/reveal.js"></script>
138 <script>
139 130
140 // More info https://github.com/hakimel/reveal.js#configuration 131 <section>
141 Reveal.initialize({ 132 <h2>Q &amp; A</h2>
142 controls: true, 133 <h3>???</h3>
143 progress: true, 134 <p><em>“Old questions are not answered—they only go out of fashion”</em> - Donald Schön</p>
144 history: true, 135 </section>
145 center: true, 136 </div>
146 transition: 'slide', // none/fade/slide/convex/concave/zoom 137 </div>
147 // More info https://github.com/hakimel/reveal.js#dependencies
148 dependencies: [
149 { src: 'lib/js/classList.js', condition: function() { return !document.body.classList; } },
150 { src: 'plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
151 { src: 'plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
152 { src: 'plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
153 { src: 'plugin/zoom-js/zoom.js', async: true },
154 { src: 'plugin/notes/notes.js', async: true }
155 ]
156 });
157 </script>
158 138
159 </body> 139 <script src="dist/reveal.js"></script>
140 <script src="plugin/zoom/zoom.js"></script>
141 <script src="plugin/notes/notes.js"></script>
142 <script src="plugin/search/search.js"></script>
143 <script src="plugin/markdown/markdown.js"></script>
144 <script src="plugin/highlight/highlight.js"></script>
145 <script>
146
147 // Also available as an ES module, see:
148 // https://revealjs.com/initialization/
149 Reveal.initialize({
150 controls: true,
151 progress: true,
152 center: true,
153 hash: true,
154
155 // Learn about plugins: https://revealjs.com/plugins/
156 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
157 });
158
159 </script>
160
161 </body>
160</html> 162</html>
diff --git a/reveal.js/flow.html b/reveal.js/flow.html
deleted file mode 100755
index c5b2ef3..0000000
--- a/reveal.js/flow.html
+++ /dev/null
@@ -1,194 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>QA flow</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 </head>
19 <body>
20 <div class="reveal">
21
22 <!-- Any section element inside of this container is displayed as a slide -->
23 <div class="slides">
24 <section>
25 <h1>QA flow</h1>
26 <h3>Idas and thoughts</h3>
27 </br>
28 <p>
29 <small>Simeon Simeonov</small>
30 </p>
31 </section>
32
33 <section>
34 <section id="fragments">
35 <h2>Agenda</h2>
36 </br>
37 <ul>
38 <span class="fragment"><li>The problem with the current QA flow</li></span>
39 <span class="fragment"><li>Alternatives</li></span>
40 <span class="fragment"><li>Suggestions</li></span>
41 <span class="fragment"><li>Q&A and discussion</li></span>
42 </ul>
43 </section>
44 </section>
45
46
47 <section>
48 <section>
49 <h4>The problem with the current QA flow</h4>
50 <p><em>“Technical debt is a concept in software development that reflects the implied cost of additional rework caused by choosing an easy (limited) solution now instead of using a better approach that would take longer.”</em> - Wikipedia</p>
51 </br>
52 <p><em>“Accumulation of technical debt is not always a choice.”</em> - Simeon Simeonov, 21st century developer</p>
53 </section>
54
55 <section>
56 <h4>The problem with the current QA flow (2)</h4>
57 </br>
58 <ul>
59 <li>Based on pull-requests (tactical)</li>
60 <li>Poor or lack of technical (strategic) design</li>
61 <li>Lack of supervision</li>
62 </ul>
63 </section>
64
65 <section>
66 <h4>The problem with the current QA flow (3)</h4>
67 </br>
68 <p>Typical challenges</p>
69 <ul>
70 <li>What entities do we need and how will we name them? (module, package, a script...)</li>
71 <li>Where will the code reside?</li>
72 <li>What is the best way to interract? (Object-oriented design in the case of Python)</li>
73 <li>Is this the best way (even in pragmatic terms)?</li>
74 <li>Any known best practices?</li>
75 </ul>
76 </section>
77
78 </section>
79
80
81 <section>
82 <section>
83 <h3>Alternatives</h3>
84 </br>
85 <p>What do others do?</p>
86 </section>
87
88 <section>
89 <h3>Alternatives (2)</h3>
90 </br>
91 <p>Google, Microsoft...</p>
92 <ul>
93 <li>Detailed technical design is expensive and requires hierarchial structure</li>
94 <li>They do not exploit our logistical advantage of being in one place</li>
95 </ul>
96 </section>
97
98 <section>
99 <h3>Alternatives (3)</h3>
100 </br>
101 <p>Linux...</p>
102 <ul>
103 <li>Brutal and hierarchial structure with a lot of overhead</li>
104 <li>They do not exploit our logistical advantage of being in one place</li>
105 </ul>
106 </section>
107
108 </section>
109
110
111 <section>
112
113 <section>
114 <h3>Suggestions</h3>
115 </br>
116 <ul>
117 <li>Automate most of the current QA tasks through better test coverage, static checking and linting</li>
118 <li>Modify the QA flow</li>
119 <li>Consider a more detailed technical specification(s)</li>
120 </ul>
121 </section>
122
123 <section>
124 <h3>Automation</h3>
125 </br>
126 <ul>
127 <li>better test coverage - as much as possible, something is better than nothing</li>
128 <li>own flake / pylint modules, as well as disabling the annoying ones as part of CI</li>
129 </ul>
130 </section>
131
132 <section>
133 <h3>New QA flow</h3>
134 </br>
135 <ol>
136 <li>Decide whether the task is "strategical" during planning and mark it accordingly in Jira</li>
137 <li>Assign supervising developer once the task is picked</li>
138 <li>Commence the design phase(*)</li>
139 <li>Approval by the supervising developer should be required</li>
140 </ol>
141 </section>
142
143 <section>
144 <h3>New QA flow (2)</h3>
145 </br>
146 <p>Design phase</p>
147 <ol>
148 <li>Identify all entities, entry points and parameters (in case of CLI)</li>
149 <li>Decide on names, naming conventions and where different enteties will reside</li>
150 <li>Decide on code entities and define the public interface (OO design)</li>
151 </ol>
152 </section>
153
154 <section>
155 <h3>A more detailed technical specification?</h3>
156 </br>
157 <p>May be feasible when developing new systems and micro-services</p>
158 </section>
159
160 </section>
161
162
163 <section>
164 <h1>Q&A</h1>
165 </section>
166
167
168 </div>
169 </div>
170
171 <script src="dist/reveal.js"></script>
172 <script src="plugin/zoom/zoom.js"></script>
173 <script src="plugin/notes/notes.js"></script>
174 <script src="plugin/search/search.js"></script>
175 <script src="plugin/markdown/markdown.js"></script>
176 <script src="plugin/highlight/highlight.js"></script>
177 <script>
178
179 // Also available as an ES module, see:
180 // https://revealjs.netlify.app/initialization/
181 Reveal.initialize({
182 controls: true,
183 progress: true,
184 center: true,
185 hash: true,
186
187 // Learn about plugins: https://revealjs.netlify.app/plugins/
188 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
189 });
190
191 </script>
192
193 </body>
194</html>
diff --git a/reveal.js/gateway.html b/reveal.js/gateway.html
deleted file mode 100644
index 4996a9c..0000000
--- a/reveal.js/gateway.html
+++ /dev/null
@@ -1,152 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>API gateway</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9
10 <meta name="viewport" content="width=device-width, initial-scale=1.0">
11
12 <link rel="stylesheet" href="dist/reset.css">
13 <link rel="stylesheet" href="dist/reveal.css">
14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- Printing and PDF exports -->
19 <script>
20 var link = document.createElement( 'link' );
21 link.rel = 'stylesheet';
22 link.type = 'text/css';
23 link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
24 document.getElementsByTagName( 'head' )[0].appendChild( link );
25 </script>
26 <!--[if lt IE 9]>
27 <script src="lib/js/html5shiv.js"></script>
28 <![endif]-->
29 </head>
30 <body>
31 <div class="reveal">
32
33 <!-- Any section element inside of this container is displayed as a slide -->
34 <div class="slides">
35 <section>
36 <h1>API gateway</h1>
37 <h3>Cerebrum-seminar 2016</h3>
38 </br>
39 <p>
40 <small>Simeon Simeonov</small>
41 </p>
42 </section>
43
44 <section>
45 <h2>Simeon Simeonov</h2>
46 </br>
47 <ul>
48 <li>
49 <p>... also known as <em>sgs</em> (<a href="http://folk.uio.no/sgs/">http://folk.uio.no/sgs/</a>)
50 </p>
51 </li>
52 <li><p>Cerebrum developer since February 2015</p></li>
53 </ul>
54 </section>
55
56
57 <section>
58 <h2>API gateway vs. API proxy</h2>
59 </br>
60 <p>The integration project at UiO (<em>INT</em>) defines the role <em>API Manager</em>. Why use API gateway instead of API proxy?</p>
61 <ul>
62 <li><p>Unified access for all our APIs</p></li>
63 <li><p>Unified documentation and catalogue</p></li>
64 <li><p>Unified caching for all APIs</p></li>
65 <li><p>Unified logging / statistics generation for all APIs</p></li>
66 <li><p>Access control / rate limiting for all our APIs</p></li>
67 <li><p>(Re)mapping and (re)writing</p></li>
68 </ul>
69 </section>
70
71
72 <section>
73 <section>
74 <h2>API gateway &amp; Cerebrum</h2>
75 </br>
76 <p>For the time being we have settled on <em>Tyk API gateway</em>.</p>
77 <p>It uses an internal user-database and consists of the following logical components:</p>
78 <ul>
79 <li><p>Dashboard</p></li>
80 <li><p>Developer portal</p></li>
81 <li><p>Gateway</p></li>
82 </ul>
83 </section>
84
85 <section>
86 <h2>Dashboard</h2>
87 <p>Provides a convenient web-interface for configuration and manipulation of the gateway and the different APIs.</p>
88 <img src="images/gateway/dashboard01.png"></img>
89
90 </section>
91
92 <section>
93 <h2>Portal</h2>
94 Provides the API documentation and catalogue.
95 <img src="images/gateway/portal01.png"></img>
96 </section>
97
98 <section>
99 <h2>Portal</h2>
100 <img src="images/gateway/portal02.png"></img>
101 </section>
102
103 <section>
104 <h2>Gataway</h2>
105 </br>
106 <ul>
107 <li><p>Still in experimental / pilot phase</p></li>
108 <li><p>Several APIs are already configured in Tyk (Cerebrum WS, SAP WS... )</p></li>
109 <li><p>Support for many <em>authentication modes</em> (Auth Token, JWT, OAuth 2, Basic Auth, Open ID Connect, ...)</p></li>
110 </ul>
111 <p>
112 Official site: <a href="https://tyk.io/">https://tyk.io/</a>
113 </p>
114 </section>
115
116 </section>
117
118
119 <section>
120 <h1>Q&amp;A</h1>
121 </section>
122
123
124 </div>
125 </div>
126
127 <script src="lib/js/head.min.js"></script>
128 <script src="js/reveal.js"></script>
129 <script>
130 // More info https://github.com/hakimel/reveal.js#configuration
131 Reveal.initialize({
132 controls: true,
133 progress: true,
134 history: true,
135 center: true,
136
137 transition: 'slide', // none/fade/slide/convex/concave/zoom
138
139 // More info https://github.com/hakimel/reveal.js#dependencies
140 dependencies: [
141 { src: 'lib/js/classList.js', condition: function() { return !document.body.classList; } },
142 { src: 'plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
143 { src: 'plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
144 { src: 'plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
145 { src: 'plugin/zoom-js/zoom.js', async: true },
146 { src: 'plugin/notes/notes.js', async: true }
147 ]
148 });
149 </script>
150
151 </body>
152</html>
diff --git a/reveal.js/mqprod.html b/reveal.js/mqprod.html
index fb3caf8..9aef4bc 100644
--- a/reveal.js/mqprod.html
+++ b/reveal.js/mqprod.html
@@ -1,248 +1,248 @@
1<!doctype html> 1<!doctype html>
2<html lang="en"> 2<html lang="en">
3 <head> 3 <head>
4 <meta charset="utf-8"> 4 <meta charset="utf-8">
5 <title>Meldingskø for UiO</title> 5 <title>Meldingskø for UiO</title>
6 <meta name="author" content="Kai Vaade, Simeon Simeonov"> 6 <meta name="author" content="Kai Vaade, Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes"> 7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"> 8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 9
10 <meta name="viewport" content="width=device-width, initial-scale=1.0"> 10 <meta name="viewport" content="width=device-width, initial-scale=1.0">
11 11
12 <link rel="stylesheet" href="dist/reset.css"> 12 <link rel="stylesheet" href="dist/reset.css">
13 <link rel="stylesheet" href="dist/reveal.css"> 13 <link rel="stylesheet" href="dist/reveal.css">
14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme"> 14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
15 15
16 <!-- Theme used for syntax highlighting of code --> 16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme"> 17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 </head> 18 </head>
19 <body> 19 <body>
20 <div class="reveal"> 20 <div class="reveal">
21 21
22 <!-- Any section element inside of this container is displayed as a slide --> 22 <!-- Any section element inside of this container is displayed as a slide -->
23 <div class="slides"> 23 <div class="slides">
24 <section> 24 <section>
25 <h2>Meldingskø for UiO</h2> 25 <h2>Meldingskø for UiO</h2>
26 <p>Utviklerforum 14.12.2017</p> 26 <p>Utviklerforum 14.12.2017</p>
27 <br/> 27 <br/>
28 <p> 28 <p>
29 <small>Kai Vaade (KIA), Simeon Simeonov (INT)</small> 29 <small>Kai Vaade (KIA), Simeon Simeonov (INT)</small>
30 </p> 30 </p>
31 </section> 31 </section>
32 32
33 <section> 33 <section>
34 <h2>Agenda</h2> 34 <h2>Agenda</h2>
35 <br/> 35 <br/>
36 <ul> 36 <ul>
37 <li>Administrasjon</li> 37 <li>Administrasjon</li>
38 <li>Roller og entiteter</li> 38 <li>Roller og entiteter</li>
39 <li>Flyt</li> 39 <li>Flyt</li>
40 <li>Rettigheter</li> 40 <li>Rettigheter</li>
41 <li>Q &amp; A</li> 41 <li>Q &amp; A</li>
42 </ul> 42 </ul>
43 </section> 43 </section>
44 44
45 <section> 45 <section>
46 <h2>Generelt</h2> 46 <h2>Generelt</h2>
47 </br> 47 </br>
48 <p>Vi bruker RabbitMQ med AMQP 0.9.1 til å utveksle / behandle meldinger (JSON, <a href="https://tools.ietf.org/html/draft-hunt-idevent-scim-00">SCIM</a>)</p> 48 <p>Vi bruker RabbitMQ med AMQP 0.9.1 til å utveksle / behandle meldinger (JSON, <a href="https://tools.ietf.org/html/draft-hunt-idevent-scim-00">SCIM</a>)</p>
49 <p>Bruk av message-broker startet i 2015. I produksjon siden høsten 2017.</p> 49 <p>Bruk av message-broker startet i 2015. I produksjon siden høsten 2017.</p>
50 <p>Driftes av KIA.</p> 50 <p>Driftes av KIA.</p>
51 </section> 51 </section>
52 52
53 <section> 53 <section>
54 <h2>Administrasjon</h2> 54 <h2>Administrasjon</h2>
55 <p>Cerebrum har brukt MQ et års tid på egen server satt opp av Seksjon for integrasjon og elektroniske identiteter (USITINT).</p> 55 <p>Cerebrum har brukt MQ et års tid på egen server satt opp av Seksjon for integrasjon og elektroniske identiteter (USITINT).</p>
56 <p>Nå er driften av tjenesten flyttet fra prosjektet og inn i linja, Gruppe for drift av katalog-, integrasjon- og autentiseringstjenester (KIA).</p> 56 <p>Nå er driften av tjenesten flyttet fra prosjektet og inn i linja, Gruppe for drift av katalog-, integrasjon- og autentiseringstjenester (KIA).</p>
57 </section> 57 </section>
58 58
59 <section> 59 <section>
60 <h2>Administrasjon (forts...)</h2> 60 <h2>Administrasjon (forts...)</h2>
61 <p>Vi har softlaunchet meldingskø-tjenesten i høst. Tjenesten er ikke "offisielt" lansert, kjører kun våre egne ting (Cerebrum/SAP) foreløpig.</p> 61 <p>Vi har softlaunchet meldingskø-tjenesten i høst. Tjenesten er ikke "offisielt" lansert, kjører kun våre egne ting (Cerebrum/SAP) foreløpig.</p>
62 <p>Antakelig gjør vi tjenesten kjent samtidig som et annet delprosjekt (API Manager) i UiO INTARK lanseres i nær fremtid.</p> 62 <p>Antakelig gjør vi tjenesten kjent samtidig som et annet delprosjekt (API Manager) i UiO INTARK lanseres i nær fremtid.</p>
63 </section> 63 </section>
64 64
65 <section> 65 <section>
66 <h2>Administrasjon (forts...)</h2> 66 <h2>Administrasjon (forts...)</h2>
67 <p>Foreløpig så har vi ikke så veldig mye erfaring med tjenesten i linja.</p> 67 <p>Foreløpig så har vi ikke så veldig mye erfaring med tjenesten i linja.</p>
68 <p>Vi vet heller ikke så mye om behovet; dvs hvor mange som kommer til å bruke tjenesten.</p> 68 <p>Vi vet heller ikke så mye om behovet; dvs hvor mange som kommer til å bruke tjenesten.</p>
69 </section> 69 </section>
70 70
71 <section> 71 <section>
72 <h2>Administrasjon (forts...)</h2> 72 <h2>Administrasjon (forts...)</h2>
73 <p>Hvordan kontakte oss:</p> 73 <p>Hvordan kontakte oss:</p>
74 <p><a href="https://www.usit.uio.no/om/organisasjon/iti/td/kia/dokumentasjon/meldingsko/" target="_blank">https://www.usit.uio.no/om/organisasjon/iti/td/kia/dokumentasjon/meldingsko/</a></p> 74 <p><a href="https://www.usit.uio.no/om/organisasjon/iti/td/kia/dokumentasjon/meldingsko/" target="_blank">https://www.usit.uio.no/om/organisasjon/iti/td/kia/dokumentasjon/meldingsko/</a></p>
75 <p>Tilgang til grensesnittet: <a href="https://mq.uio.no/">https://mq.uio.no</a></p> 75 <p>Tilgang til grensesnittet: <a href="https://mq.uio.no/">https://mq.uio.no</a></p>
76 </section> 76 </section>
77 77
78 <section> 78 <section>
79 <h2>Roller og entiteter</h2> 79 <h2>Roller og entiteter</h2>
80 <ul> 80 <ul>
81 <p>Roller:</p> 81 <p>Roller:</p>
82 <li>Administrator / Manager</li> 82 <li>Administrator / Manager</li>
83 <li>Consumer / Konsument</li> 83 <li>Consumer / Konsument</li>
84 <li>Publisher / Publisist :)</li> 84 <li>Publisher / Publisist :)</li>
85 <p>Entiteter:</p> 85 <p>Entiteter:</p>
86 <li>vhost</li> 86 <li>vhost</li>
87 <li>Exchange (direct, fanout, topic, ...)</li> 87 <li>Exchange (direct, fanout, topic, ...)</li>
88 <li>Kø (transient, durable)</li> 88 <li>Kø (transient, durable)</li>
89 <li>Binding</li> 89 <li>Binding</li>
90 </ul> 90 </ul>
91 </section> 91 </section>
92 92
93 <section> 93 <section>
94 <h2>Flyt</h2> 94 <h2>Flyt</h2>
95 </br> 95 </br>
96 <img src="images/mqprod/flow01.png"></img> 96 <img src="images/mqprod/flow01.png"></img>
97 </section> 97 </section>
98 98
99 <section> 99 <section>
100 <h2>Rettigheter</h2> 100 <h2>Rettigheter</h2>
101 <p>RabbitMQ implementerer 2 rettighetsnivåer: per <em>vhost</em> og per entitet</p> 101 <p>RabbitMQ implementerer 2 rettighetsnivåer: per <em>vhost</em> og per entitet</p>
102 <p>RabbitMQ (AMQP) definerer 3 typer operasjoner:</p> 102 <p>RabbitMQ (AMQP) definerer 3 typer operasjoner:</p>
103 <ul> 103 <ul>
104 <li><p><em>configure</em> - opprette / slette entiteter eller endre deres oppførsel</p></li> 104 <li><p><em>configure</em> - opprette / slette entiteter eller endre deres oppførsel</p></li>
105 <li><p><em>write</em> - skrive melding til en entitet</p></li> 105 <li><p><em>write</em> - skrive melding til en entitet</p></li>
106 <li><p><em>read</em> - lese melding fra entitet</p></li> 106 <li><p><em>read</em> - lese melding fra entitet</p></li>
107 </ul> 107 </ul>
108 </section> 108 </section>
109 109
110 <section> 110 <section>
111 <h2>Rettigheter og brukere</h2> 111 <h2>Rettigheter og brukere</h2>
112 <p>RabbitMQ bruker <em>regular expressions</em> til å definere rettigheter</p> 112 <p>RabbitMQ bruker <em>regular expressions</em> til å definere rettigheter</p>
113 <ul> 113 <ul>
114 <li><p><small><em>rabbitmqctl add_user cerebrum &lt;passord&gt;</em></small></p></li> 114 <li><p><small><em>rabbitmqctl add_user cerebrum &lt;passord&gt;</em></small></p></li>
115 <li><p><small><em>rabbitmqctl add_user uio_ad_microservice &lt;passord&gt;</em></small></p></li> 115 <li><p><small><em>rabbitmqctl add_user uio_ad_microservice &lt;passord&gt;</em></small></p></li>
116 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"</em></small></p></li> 116 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"</em></small></p></li>
117 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"</em></small></p></li> 117 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"</em></small></p></li>
118 </ul> 118 </ul>
119 </section> 119 </section>
120 120
121 <section> 121 <section>
122 <h2><em>Routing keys</em> i <em>topic exchange</em></h2> 122 <h2><em>Routing keys</em> i <em>topic exchange</em></h2>
123 <p><em>Routing key</em> settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en <em>topic exchange</em>.</p> 123 <p><em>Routing key</em> settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en <em>topic exchange</em>.</p>
124 <p>Strukturen til en topic / message routing key er:</p> 124 <p>Strukturen til en topic / message routing key er:</p>
125 <p><small>&lt;kilde&gt;.&lt;type&gt;.&lt;objekt&gt;.&lt;hendelse&gt;</small></p> 125 <p><small>&lt;kilde&gt;.&lt;type&gt;.&lt;objekt&gt;.&lt;hendelse&gt;</small></p>
126 <p>F.eks.:</p> 126 <p>F.eks.:</p>
127 <p><small>cerebrum.event.person.delete</small></p> 127 <p><small>cerebrum.event.person.delete</small></p>
128 </section> 128 </section>
129 129
130 <section> 130 <section>
131 <h3>Dokumentasjon og lenker</h3> 131 <h3>Dokumentasjon og lenker</h3>
132 <p> 132 <p>
133 RabbitMQ: 133 RabbitMQ:
134 </p> 134 </p>
135 <ul> 135 <ul>
136 <li> 136 <li>
137 <p> 137 <p>
138 <small> 138 <small>
139 <a href="http://www.rabbitmq.com">http://www.rabbitmq.com</a> - RabbitMQ sin offisielle side 139 <a href="http://www.rabbitmq.com">http://www.rabbitmq.com</a> - RabbitMQ sin offisielle side
140 </small> 140 </small>
141 </li> 141 </li>
142 <li> 142 <li>
143 <p> 143 <p>
144 <small> 144 <small>
145 <a href="http://www.rabbitmq.com/configure.html">http://www.rabbitmq.com/configure.html</a> - Generell konfigurasjon av RabbitMQ 145 <a href="http://www.rabbitmq.com/configure.html">http://www.rabbitmq.com/configure.html</a> - Generell konfigurasjon av RabbitMQ
146 </small> 146 </small>
147 </li> 147 </li>
148 <li> 148 <li>
149 <p> 149 <p>
150 <small> 150 <small>
151 <a href="http://www.rabbitmq.com/management.html">http://www.rabbitmq.com/management.html</a> - Bruk av rabbitmq_management plugin i RabbitMQ 151 <a href="http://www.rabbitmq.com/management.html">http://www.rabbitmq.com/management.html</a> - Bruk av rabbitmq_management plugin i RabbitMQ
152 </small> 152 </small>
153 </li> 153 </li>
154 <li> 154 <li>
155 <p> 155 <p>
156 <small> 156 <small>
157 <a href="http://www.rabbitmq.com/access-control.html">http://www.rabbitmq.com/access-control.html</a> - Tilgangskontroll i RabbitMQ 157 <a href="http://www.rabbitmq.com/access-control.html">http://www.rabbitmq.com/access-control.html</a> - Tilgangskontroll i RabbitMQ
158 </small> 158 </small>
159 </li> 159 </li>
160 <li> 160 <li>
161 <p> 161 <p>
162 <small> 162 <small>
163 <a href="http://www.rabbitmq.com/man/rabbitmqctl.1.man.html">http://www.rabbitmq.com/man/rabbitmqctl.1.man.html</a> - rabbitmqctl manual side 163 <a href="http://www.rabbitmq.com/man/rabbitmqctl.1.man.html">http://www.rabbitmq.com/man/rabbitmqctl.1.man.html</a> - rabbitmqctl manual side
164 </small> 164 </small>
165 </li> 165 </li>
166 <li> 166 <li>
167 <p> 167 <p>
168 <small> 168 <small>
169 <a href="http://www.rabbitmq.com/tutorials/amqp-concepts.html">http://www.rabbitmq.com/tutorials/amqp-concepts.html</a> - Generelt om AMQP og forskjellige exchanges 169 <a href="http://www.rabbitmq.com/tutorials/amqp-concepts.html">http://www.rabbitmq.com/tutorials/amqp-concepts.html</a> - Generelt om AMQP og forskjellige exchanges
170 </small> 170 </small>
171 </li> 171 </li>
172 </ul> 172 </ul>
173 </section> 173 </section>
174 174
175 <section> 175 <section>
176 <h3>Dokumentasjon og lenker</h3> 176 <h3>Dokumentasjon og lenker</h3>
177 <p> 177 <p>
178 UiO: 178 UiO:
179 </p> 179 </p>
180 <ul> 180 <ul>
181 <li> 181 <li>
182 <p> 182 <p>
183 <small> 183 <small>
184 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/dokumentasjon/tools/rabbitmq.html">Installasjon og oppsett av RabbitMQ for Cerebrum utviklere</a> 184 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/dokumentasjon/tools/rabbitmq.html">Installasjon og oppsett av RabbitMQ for Cerebrum utviklere</a>
185 </small> 185 </small>
186 </li> 186 </li>
187 <li> 187 <li>
188 <p> 188 <p>
189 <small> 189 <small>
190 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/aktiviteter/office365-uia/broker/rabbitmq.html">Oppsett av RabbitMQ for Office365 integrasjon mot UiA</a> 190 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/aktiviteter/office365-uia/broker/rabbitmq.html">Oppsett av RabbitMQ for Office365 integrasjon mot UiA</a>
191 </small> 191 </small>
192 </li> 192 </li>
193 <li> 193 <li>
194 <p> 194 <p>
195 <small> 195 <small>
196 <a href="https://www.usit.uio.no/om/organisasjon/uav/usitint/interndokumentasjon/integrasjon/broker/utredning-meldingsflyt.html">Utredning: Meldingsflyt, kø-struktur og navngiving</a> (intern) 196 <a href="https://www.usit.uio.no/om/organisasjon/uav/usitint/interndokumentasjon/integrasjon/broker/utredning-meldingsflyt.html">Utredning: Meldingsflyt, kø-struktur og navngiving</a> (intern)
197 </small> 197 </small>
198 </li> 198 </li>
199 </ul> 199 </ul>
200 <p> 200 <p>
201 Bøker: 201 Bøker:
202 </p> 202 </p>
203 <ul> 203 <ul>
204 <li> 204 <li>
205 <p> 205 <p>
206 <small> 206 <small>
207 RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning 207 RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning
208 </small> 208 </small>
209 </li> 209 </li>
210 <li> 210 <li>
211 <p> 211 <p>
212 <small> 212 <small>
213 Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing 213 Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing
214 </small> 214 </small>
215 </li> 215 </li>
216 </section> 216 </section>
217 217
218 <section> 218 <section>
219 <h1>Q &amp; A</h1> 219 <h1>Q &amp; A</h1>
220 </section> 220 </section>
221 221
222 </div> 222 </div>
223 </div> 223 </div>
224 224
225 <script src="dist/reveal.js"></script> 225 <script src="dist/reveal.js"></script>
226 <script src="plugin/zoom/zoom.js"></script> 226 <script src="plugin/zoom/zoom.js"></script>
227 <script src="plugin/notes/notes.js"></script> 227 <script src="plugin/notes/notes.js"></script>
228 <script src="plugin/search/search.js"></script> 228 <script src="plugin/search/search.js"></script>
229 <script src="plugin/markdown/markdown.js"></script> 229 <script src="plugin/markdown/markdown.js"></script>
230 <script src="plugin/highlight/highlight.js"></script> 230 <script src="plugin/highlight/highlight.js"></script>
231 <script> 231 <script>
232 232
233 // Also available as an ES module, see: 233 // Also available as an ES module, see:
234 // https://revealjs.netlify.app/initialization/ 234 // https://revealjs.com/initialization/
235 Reveal.initialize({ 235 Reveal.initialize({
236 controls: true, 236 controls: true,
237 progress: true, 237 progress: true,
238 center: true, 238 center: true,
239 hash: true, 239 hash: true,
240 240
241 // Learn about plugins: https://revealjs.netlify.app/plugins/ 241 // Learn about plugins: https://revealjs.com/plugins/
242 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ] 242 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
243 }); 243 });
244 244
245 </script> 245 </script>
246 246
247 </body> 247 </body>
248</html> 248</html>
diff --git a/reveal.js/postgresql_tuning.html b/reveal.js/postgresql_tuning.html
index d7e3466..8678513 100644
--- a/reveal.js/postgresql_tuning.html
+++ b/reveal.js/postgresql_tuning.html
@@ -1,152 +1,151 @@
1<!doctype html> 1<!doctype html>
2<html lang="en"> 2<html lang="en">
3 <head> 3 <head>
4 <meta charset="utf-8"> 4 <meta charset="utf-8">
5 <title>SQLAlchemy</title> 5 <title>SQLAlchemy</title>
6 <meta name="author" content="Simeon Simeonov"> 6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes"> 7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"> 8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0"> 9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 10
11 <link rel="stylesheet" href="dist/reset.css"> 11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css"> 12 <link rel="stylesheet" href="dist/reveal.css">
13 13
14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme"> 14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme">
15 15
16 <!-- Theme used for syntax highlighting of code --> 16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme"> 17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> --> 18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head> 19 </head>
20 <body> 20 <body>
21 <div class="reveal"> 21 <div class="reveal">
22 22
23 <!-- Any section element inside of this container is displayed as a slide --> 23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides"> 24 <div class="slides">
25 25
26 <section> 26 <section>
27 <h2>PostgreSQL tuning</h2> 27 <h2>PostgreSQL tuning</h2>
28 <h4>Data Science @ Beryl</h4> 28 <h4>Data Science @ Beryl</h4>
29 </br> 29 </br>
30 <p><small>Simeon Simeonov</small></p> 30 <p><small>Simeon Simeonov</small></p>
31 </section> 31 </section>
32 32
33 <section> 33 <section>
34 34
35 <section id="fragments"> 35 <section id="fragments">
36 <h2>Agenda</h2> 36 <h2>Agenda</h2>
37 </br> 37 </br>
38 <ul> 38 <ul>
39 <span class="fragment"><li>Generic tools for gathering information</li></span> 39 <span class="fragment"><li>Generic tools for gathering information</li></span>
40 <span class="fragment"><li>Memory settings</li></span> 40 <span class="fragment"><li>Memory settings</li></span>
41 <span class="fragment"><li>Logging and performance reports</li></span> 41 <span class="fragment"><li>Logging and performance reports</li></span>
42 <span class="fragment"><li>Other tools for analysis</li></span> 42 <span class="fragment"><li>Other tools for analysis</li></span>
43 </ul> 43 </ul>
44 </section> 44 </section>
45 45
46 </section> 46 </section>
47 47
48 <section> 48 <section>
49 <h2>General tools for gathering information</h2> 49 <h2>General tools for gathering information</h2>
50 </br> 50 </br>
51 <pre data-id="code-animation"> 51 <pre data-id="code-animation">
52 <code class="bash" data-trim type="text/template"> 52 <code class="bash" data-trim type="text/template">
53 # shell 53 # shell
54 # fetch information from the OS 54 # fetch information from the OS
55 cat /proc/cpuinfo 55 cat /proc/cpuinfo
56 cat /proc/meminfo 56 cat /proc/meminfo
57 sysctl -a | grep shm # get kernel parameters of interest 57 sysctl -a | grep shm # get kernel parameters of interest
58 </code> 58 </code>
59 <code class="sql" data-trim type="text/template"> 59 <code class="sql" data-trim type="text/template">
60 -- SQL 60 -- SQL
61 -- show the current values of all settings 61 -- show the current values of all settings
62 SHOW ALL; 62 SHOW ALL;
63 63
64 -- display even more than all... 64 -- display even more than all...
65 SELECT * FROM pg_settings; 65 SELECT * FROM pg_settings;
66 66
67 -- opening postgresql.conf and reading the comments - the old school approach 67 -- opening postgresql.conf and reading the comments - the old school approach
68 </code> 68 </code>
69 <code class="bash" data-trim type="text/template"> 69 <code class="bash" data-trim type="text/template">
70 # old school: edit postgresql.conf and read the comments 70 # old school: edit postgresql.conf and read the comments
71 </code> 71 </code>
72 </pre> 72 </pre>
73 </section> 73 </section>
74 74
75 <section> 75 <section>
76 <h2>Memory settings</h2> 76 <h2>Memory settings</h2>
77 </br> 77 </br>
78 <ul> 78 <ul>
79 <li><em>shared_buffers</em> - how much memory is dedicated to PostgreSQL to use for caching data - for a system with 1GB or more of RAM, a reasonable starting value for <em>shared_buffers</em> is 25% of the system memory (128MB -> 1GB)</li> 79 <li><em>shared_buffers</em> - how much memory is dedicated to PostgreSQL to use for caching data - for a system with 1GB or more of RAM, a reasonable starting value for <em>shared_buffers</em> is 25% of the system memory (128MB -> 1GB)</li>
80 <li><em>effective_cache_size</em> - how much memory we expect to be available in the OS and PostgreSQL buffer caches, not an allocation - used only by the PostgreSQL query planner to figure out whether plans it's considering would be expected to fit in RAM or not - 1/2 of total memory would be a normal conservative setting (4GB -> 8GB)</li> 80 <li><em>effective_cache_size</em> - how much memory we expect to be available in the OS and PostgreSQL buffer caches, not an allocation - used only by the PostgreSQL query planner to figure out whether plans it's considering would be expected to fit in RAM or not - 1/2 of total memory would be a normal conservative setting (4GB -> 8GB)</li>
81 <li><em>work_mem</em> - the base maximum amount of memory to be used by a query operation (such as a sort or hash table) before writing to temporary disk files - for a complex query, several sort or hash operations might be running in parallel; each operation will generally be allowed to use as much memory as this value specifies (4MB -> 8MB)</li> 81 <li><em>work_mem</em> - the base maximum amount of memory to be used by a query operation (such as a sort or hash table) before writing to temporary disk files - for a complex query, several sort or hash operations might be running in parallel; each operation will generally be allowed to use as much memory as this value specifies (4MB -> 8MB)</li>
82 <li><em>maintenance_work_mem</em> - the maximum amount of memory to be used by maintenance operations, such as <em>VACUUM</em> and <em>CREATE INDEX</em>. It's safe to set this value significantly larger than <em>work_mem</em> (64MB -> 256MB)</li> 82 <li><em>maintenance_work_mem</em> - the maximum amount of memory to be used by maintenance operations, such as <em>VACUUM</em> and <em>CREATE INDEX</em>. It's safe to set this value significantly larger than <em>work_mem</em> (64MB -> 256MB)</li>
83 </ul> 83 </ul>
84 </section> 84 </section>
85 85
86 <section> 86 <section>
87 <h2>Logging and performance reports</h2> 87 <h2>Logging and performance reports</h2>
88 </br> 88 </br>
89 <p><em>pgBadger - a fast PostgreSQL log analysis report</em> can be used for general analysis.</p> 89 <p><em>pgBadger - a fast PostgreSQL log analysis report</em> can be used for general analysis.</p>
90 <ul> 90 <ul>
91 <li><em>log_checkpoints</em> - checkpoints and restartpoints are logged in the server log. Some statistics are included in the log messages, including the number of buffers written and the time spent writing them</li> 91 <li><em>log_checkpoints</em> - checkpoints and restartpoints are logged in the server log. Some statistics are included in the log messages, including the number of buffers written and the time spent writing them</li>
92 <li><em>log_connections</em> - each attempted connection to the server to be logged, as well as successful completion of client authentication</li> 92 <li><em>log_connections</em> - each attempted connection to the server to be logged, as well as successful completion of client authentication</li>
93 <li><em>log_disconnections</em> - provides information similar to <em>log_connections</em>, plus the duration of the session</li> 93 <li><em>log_disconnections</em> - provides information similar to <em>log_connections</em>, plus the duration of the session</li>
94 <li><em>log_line_prefix</em> - set to '%t [%p]: user=%u,db=%d,app=%a,client=%h '</li> 94 <li><em>log_line_prefix</em> - set to '%t [%p]: user=%u,db=%d,app=%a,client=%h '</li>
95 <li><em>log_lock_waits</em> - log message is produced when a session waits longer than deadlock_timeout to acquire a lock. This is useful in determining if lock waits are causing poor performance</li> 95 <li><em>log_lock_waits</em> - log message is produced when a session waits longer than deadlock_timeout to acquire a lock. This is useful in determining if lock waits are causing poor performance</li>
96 <li><em>log_temp_files</em> - when set to <em>0</em>, a log entry is emitted for each temporary file when it is deleted</li> 96 <li><em>log_temp_files</em> - when set to <em>0</em>, a log entry is emitted for each temporary file when it is deleted</li>
97 <li><em>log_autovacuum_min_duration</em> - set to <em>0</em> it logs all autovacuum actions</li> 97 <li><em>log_autovacuum_min_duration</em> - set to <em>0</em> it logs all autovacuum actions</li>
98 </ul> 98 </ul>
99 </section> 99 </section>
100 100
101 <section> 101 <section>
102 <h2>Other tools for analysis</h2> 102 <h2>Other tools for analysis</h2>
103 </br> 103 </br>
104 <ul> 104 <ul>
105 <li><em>ANALYZE</em> - collects statistics about the contents of tables in the database, and stores the results in the pg_statistic system catalog. Subsequently, the query planner uses these statistics to help determine the most efficient execution plans for queries.</li> 105 <li><em>ANALYZE</em> - collects statistics about the contents of tables in the database, and stores the results in the pg_statistic system catalog. Subsequently, the query planner uses these statistics to help determine the most efficient execution plans for queries.</li>
106 <li><em>VACUUM</em> - reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. (<em>VACUUM</em> vs. <em>VACUUM FULL</em>)</li> 106 <li><em>VACUUM</em> - reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. (<em>VACUUM</em> vs. <em>VACUUM FULL</em>)</li>
107 </ul> 107 </ul>
108 </section> 108 </section>
109 109
110 <section> 110 <section>
111 <h2>Sources</h2> 111 <h2>Sources</h2>
112 </br> 112 </br>
113 <p> 113 <p>
114 <a href="https://www.postgresql.org/docs/">https://www.postgresql.org/docs/</a> - The official documentation 114 <a href="https://www.postgresql.org/docs/">https://www.postgresql.org/docs/</a> - The official documentation
115 </p> 115 </p>
116 <p> 116 <p>
117 <a href="https://wiki.postgresql.org">https://wiki.postgresql.org</a> - The official Wiki 117 <a href="https://wiki.postgresql.org">https://wiki.postgresql.org</a> - The official Wiki
118 </p> 118 </p>
119 </section> 119 </section>
120 120
121 <section> 121 <section>
122 <h1>Q &amp; A</h1> 122 <h1>Q &amp; A</h1>
123 </section> 123 </section>
124 124
125 </div> 125 </div>
126 </div> 126 </div>
127
128 <script src="dist/reveal.js"></script>
129 <script src="plugin/zoom/zoom.js"></script>
130 <script src="plugin/notes/notes.js"></script>
131 <script src="plugin/search/search.js"></script>
132 <script src="plugin/markdown/markdown.js"></script>
133 <script src="plugin/highlight/highlight.js"></script>
134 <script>
135 127
136 // Also available as an ES module, see: 128 <script src="dist/reveal.js"></script>
137 // https://revealjs.netlify.app/initialization/ 129 <script src="plugin/zoom/zoom.js"></script>
138 Reveal.initialize({ 130 <script src="plugin/notes/notes.js"></script>
139 controls: true, 131 <script src="plugin/search/search.js"></script>
140 progress: true, 132 <script src="plugin/markdown/markdown.js"></script>
141 center: true, 133 <script src="plugin/highlight/highlight.js"></script>
142 hash: true, 134 <script>
143 135
144 // Learn about plugins: https://revealjs.netlify.app/plugins/ 136 // Also available as an ES module, see:
145 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ] 137 // https://revealjs.com/initialization/
146 }); 138 Reveal.initialize({
147 Reveal.configure({ pdfSeparateFragments: false }); 139 controls: true,
140 progress: true,
141 center: true,
142 hash: true,
148 143
149 </script> 144 // Learn about plugins: https://revealjs.com/plugins/
145 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
146 });
150 147
151 </body> 148 </script>
149
150 </body>
152</html> 151</html>
diff --git a/reveal.js/python-oo.html b/reveal.js/python-oo.html
deleted file mode 100644
index 5279993..0000000
--- a/reveal.js/python-oo.html
+++ /dev/null
@@ -1,145 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>Object-oriented programming in Python</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/statnett.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>Object-oriented programming in Python</h2>
28 <h4>Data Engineering @ Statnett</h4>
29 </br>
30 <p><small>Simeon Simeonov</small></p>
31 </section>
32
33 <section>
34 <h2>Goals</h2>
35 </br>
36 <ul>
37 <li><p>Present the Python programming language in a different way than <a href="https://docs.python.org">https://docs.python.org</a></p></li>
38 <li><p>Avoid information overload</p></li>
39 <li><p>Use examples and interaction rather than documents and slides</p></li>
40 </ul>
41 </section>
42
43 <section>
44 <h2>Preliminary plan</h2>
45 </br>
46 <ul>
47 <li><p>Basics: About the language, the Python eco-system, types, modules, functions, scopes, decorators, string formatting</p></li>
48 <li><p><b>Object-oriented programming in Python: How Python "really works"</b></p></li>
49 <li><p>Control flow: if / for / while / try, iterators, "tactical programming" tips</p></li>
50 <li><p>A brief tour through Python's standard library</p></li>
51 <li><p>Code design and best practices: How to design your code</p></li>
52 </ul>
53 </section>
54
55 <section>
56 <h2>What is object-oriented programming?</h2>
57 </br>
58 <p>Object-oriented programming does <b>not</b> mean using an object-oriented programming language.</p>
59 <p>It is rather a programming paradigm based on the concept of <em>object</em>, as well as on some general principles and best practices aiming at:</p>
60 <ul>
61 <li><p>improving readability</p></li>
62 <li><p>improving re-usability</p></li>
63 <li><p>improving modularity</p></li>
64 <li><p>providing foundation for a more intuitive design</p></li>
65 </ul>
66 </section>
67
68 <section>
69 <h2>General principles</h2>
70 </br>
71 <p>The following four concepts / principles are presented in most object-oriented programming books:</p>
72 <ul>
73 <li><p>separate and hide "private" details from the outside world and / or child (inheriting) functionality (encapsulation)</p></li>
74 <li><p>separate the interface from its implementation (abstraction)</p></li>
75 <li><p>inherit and extend / adapt existing functionality, through "is-a" relationship hierarchy (inheritance)</p></li>
76 <li><p>execute different code / functionality based on the object's place in the hierarchy (polymorphism)</p></li>
77 </ul>
78 </section>
79
80 <section>
81 <h2>Building blocks and definitions</h2>
82 </br>
83 <p><b>Note:</b> "Lacking universally accepted terminology to talk about classes, I will make occasional use of Smalltalk and C++ terms." - The Python tutorial (https://docs.python.org)</p>
84 <p>When talking about object-oriented programming, the following building blocks are involved:</p>
85 <ul>
86 <li><p>class - a blueprint / template for creating objects</p></li>
87 <li><p>object - an instance of a class that may contain its own attributes as well as references to its class' attributes</p></li>
88 <li><p>attribute - variable, property, function defined in the class and present in its instances</p></li>
89 <li><p>class variable - attribute of which a single copy exists, regardless of how many instances of the class exist</p></li>
90 <li><p>object / instance variable - attribute for which each instantiated object of the class has a separate copy, or instance</p></li>
91 <li><p>method - member function - function that is an attribute</p></li>
92 </ul>
93 </section>
94
95 <section>
96 <h2>The object-oriented world of Python</h2>
97 </br>
98 <p>Some additional concepts and definitions are introduced in Python, as well as in some other programming languages:</p>
99 <ul>
100 <li><p>metaclass - a class whose instances are classes</p></li>
101 <li><p>property - attribute that provides a flexible mechanism to read, write, or compute the value of a "private" attribute</p></li>
102 <li><p>type - old C object-oriented term, now (in Python 3) considered to be the same as class</p></li>
103 </ul>
104 <p> As is true for modules, classes partake of the dynamic nature of Python: they are created at runtime, and can be modified further after creation.</p>
105 </section>
106
107 <section>
108 <h2>The object-oriented world of Python (cont...)</h2>
109 </br>
110 <p>The < </p>
111 <ul>
112 <li><p>metaclass - a class whose instances are classes</p></li>
113 <li><p>property - attribute that provides a flexible mechanism to read, write, or compute the value of a "private" attribute</p></li>
114 <li><p>type - old C object-oriented term, now (in Python 3) considered to be the same as class</p></li>
115 </ul>
116 </section>
117
118 </div>
119 </div>
120
121 <script src="dist/reveal.js"></script>
122 <script src="plugin/zoom/zoom.js"></script>
123 <script src="plugin/notes/notes.js"></script>
124 <script src="plugin/search/search.js"></script>
125 <script src="plugin/markdown/markdown.js"></script>
126 <script src="plugin/highlight/highlight.js"></script>
127 <script>
128
129 // Also available as an ES module, see:
130 // https://revealjs.netlify.app/initialization/
131 Reveal.initialize({
132 controls: true,
133 progress: true,
134 center: true,
135 hash: true,
136
137 // Learn about plugins: https://revealjs.netlify.app/plugins/
138 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
139 });
140 Reveal.configure({ pdfSeparateFragments: false });
141
142 </script>
143
144 </body>
145</html>
diff --git a/reveal.js/python.html b/reveal.js/python.html
deleted file mode 100644
index d27cd49..0000000
--- a/reveal.js/python.html
+++ /dev/null
@@ -1,680 +0,0 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>Introduction to Python</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/statnett.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>Introduction to Python</h2>
28 <h4>Data Engineering @ Statnett</h4>
29 </br>
30 <p><small>Simeon Simeonov</small></p>
31 </section>
32
33 <section>
34 <h2>Goals</h2>
35 </br>
36 <ul>
37 <li><p>Present the Python programming language in a different way than <a href="https://docs.python.org">https://docs.python.org</a></p></li>
38 <li><p>Avoid information overload</p></li>
39 <li><p>Use examples and interaction rather than documents and slides</p></li>
40 </ul>
41 </section>
42
43 <section>
44 <h2>Preliminary plan</h2>
45 </br>
46 <ul>
47 <li><p><b>Basics: About the language, the Python eco-system, types, modules, functions, scopes, decorators, string formatting</b></p></li>
48 <li><p>Object-oriented programming in Python: How Python "really works"</p></li>
49 <li><p>Control flow: if / for / while / try, iterators, "tactical programming" tips</p></li>
50 <li><p>A brief tour through Python's standard library</p></li>
51 <li><p>Code design and best practices: How to design your code</p></li>
52 </ul>
53 </section>
54
55 <section>
56 <h2>What is Python?</h2>
57 </br>
58 <ul>
59 <li><p>Python is an interpreted high-level general-purpose programming language - advanced through the Python Enhancement Proposal (PEP) process</p></li>
60 <li><p>CPython is the reference implementation of Python, written in C (alternatives: pypy, jython)</p></li>
61 <li><p>python - interpreter and interpreter shell (alternatives: ipython, bpython)</p></li>
62 <li><p>libpython</p></li>
63 <li><p>Calling C from Python: Cython, CFFI, ctypes</p></li>
64 </ul>
65 </section>
66
67 <section>
68 <h2>Philosophy</h2>
69 </br>
70 <pre data-id="code-animation">
71 <code class="bash" data-trim type="text/template">
72 $ python
73 </code>
74 </pre>
75 <pre data-id="code-animation">
76 <code class="python" data-trim type="text/template">
77 import this
78
79 # The Zen of Python, by Tim Peters
80
81 # Beautiful is better than ugly.
82 # Explicit is better than implicit.
83 # Simple is better than complex.
84 # Complex is better than complicated.
85 # Flat is better than nested.
86 # Sparse is better than dense.
87 # Readability counts.
88 # Special cases aren't special enough to break the rules.
89 # Although practicality beats purity.
90 # Errors should never pass silently.
91 # Unless explicitly silenced.
92 # In the face of ambiguity, refuse the temptation to guess.
93 # There should be one-- and preferably only one --obvious way to do it.
94 # Although that way may not be obvious at first unless you're Dutch.
95 # Now is better than never.
96 # Although never is often better than *right* now.
97 # If the implementation is hard to explain, it's a bad idea.
98 # If the implementation is easy to explain, it may be a good idea.
99 # Namespaces are one honking great idea -- let's do more of those!
100 </code>
101 </pre>
102 </section>
103
104 <section>
105 <h2>Built-in functions</h2>
106 </br>
107 <p>Few built-in functions.</p>
108 <p><a href="https://docs.python.org/3/library/functions.html">https://docs.python.org/3/library/functions.html</a></p>
109 <ul>
110 <li><em>dir([obj])</em> - returns a list of valid attributes for that object</li>
111 <li><em>id(obj)</em> - returns the "identity" of an object - an integer which is guaranteed to be unique</li>
112 <li><em>print(...)</em> - prints objects to a text stream</li>
113 <li><em>str(...)</em> - returns a string version of object</li>
114 <li><em>type(obj)</em> - returns the type of an object</li>
115 </ul>
116 </section>
117
118 <section>
119 <h2>Common built-in types</h2>
120 </br>
121 <p>Python uses <a href="https://en.wikipedia.org/wiki/Duck_typing"><em>duck typing</em></a> and has typed objects but untyped variable names.</p>
122 <p>Type constraints are not checked at compile time; rather, operations on an object may fail, signifying that the given object is not of a suitable type. Despite being dynamically-typed, Python is strongly-typed, forbidding operations that are not well-defined (for example, adding a number to a string) rather than silently attempting to make sense of them.</p>
123 <pre data-id="code-animation">
124 <code class="python" data-trim type="text/template">
125 s = 'foo' # this is a string / str, same as str('foo'), may be encoded, immutable
126 b = b'foo' # bytes, same as bytes('foo', 'utf-8'), may be decoded, immutable
127 i = 6 # int, same as int('6'), immutable
128 f = 0.1 # float, same as float('0.1'), immutable
129 b = False # bool, same as bool(0), bool(''), bool(None)... immutable / constant
130 n = None # NoneType, similar to 'null' in other languages, immutable / constant
131 l = [1, False, 'foo'] # list, same as list((1, False, 'foo'))
132 t = (1, False, 'foo') # tuple, same as tuple([1, False, 'foo']), immutable
133 d = {'foo': 1, 'bar': 8} # dict, same as dict(foo=1, bar=8), similar to hash in other languages
134 s = {'foo', 'bar', 1, 1, 4} # set, same as set(['foo', 'bar', 1, 1, 4]), removes duplicates
135 </code>
136 </pre>
137 <p>Classes, functions, modules, .... and even types are simply other types :)</p>
138 </section>
139
140 <section>
141 <h2>Modules</h2>
142 </br>
143 <p>A module is a file containing Python definitions and statements. The file name is the module name with the suffix <em>.py</em> appended. Within a module, the module's name (as a string) is available as the value of the global variable <em>__name__</em></p>
144 <p>When a module named <em>foo</em> is imported, the interpreter first searches for a built-in module with that name (<em>sys.builtin_module_names</em>). If not found, it then searches for a file named <em>foo.py</em> in a list of directories given by the variable <em>sys.path</em>. <em>sys.path</em> is initialized from these locations:</p>
145 <ul>
146 <li>the directory containing the input script (or the current directory when no file is specified)</li>
147 <li><em>PYTHONPATH</em> - env. variable - a list of directory names</li>
148 <li>the installation-dependent default locations</li>
149 </ul>
150
151 <p>The module is then imported only once and "cached" in <em>sys.modules</em></p>
152 </section>
153
154 <section>
155 <h2>Packages</h2>
156 </br>
157 <p><a href="https://docs.python.org/3/tutorial/modules.html#packages">Packages</a> are a way of structuring Python's module namespace by using "dotted module names"</p>
158 <p>The import statement combines two operations:</p>
159 <ul>
160 <li>it searches for the named module</li>
161 <li>it binds the results of that search to a name in the <em>local scope</em></li>
162 </ul>
163 <pre data-id="code-animation">
164 <code class="python" data-trim type="text/template">
165 # bar.py then bar/__init__.py will be considered, the first match executed and bound to 'bar'
166 import bar
167
168 import mymodule.foo # implicitly executes mymodule.py, mymodule/__init__.py and mymodule/foo/__init__.py
169
170 import numpy as np # will be bound as 'np' instead of 'numpy'. N.B. __name__ is still 'numpy'
171
172 import some.extremely.deep.path.Animal as Animal # "sacrifice" the namespace in the name of convinience
173
174 from sys import path # execute sys and only import the 'path' attribute into local scope as 'path'
175
176 # relative imports must be explicit in Python 3
177 from .othermodule import something # expects that current module and 'othermodule' are in the same
178 # package (containing __init__.py)
179
180 from sys import * # NO! Bad programming practice since 1879
181 </code>
182 </pre>
183 </section>
184
185 <section>
186 <h2>Creating and maintaining a Python environment</h2>
187 </br>
188 <p>Python's official package repository is <a href="https://pypi.org"><em>PyPi (https://pypi.org)</em></a>, while Python's official package installer is <a href="https://pypi.org/project/pip/"><em>pip (https://pypi.org/project/pip/)</em></a></p>
189 <p>A Python environment is the physical and logical arrangement of Python modules and packages. Several options exist:</p>
190 <ul>
191 <li>using a proper operating system :) (symlinks, real commercial support etc.)</li>
192 <li>using <em>venv</em></li>
193 <li>using higher level tools like <em>poetry</em></li>
194 <li>using a mixture / cocktail of all of the above :)</li>
195 </ul>
196 </section>
197
198 <section>
199 <h2>Creating and maintaining a Python environment (cont...)</h2>
200 </br>
201 <p>Desired qualities for a flexible Python environment:</p>
202 <ul>
203 <li>easy to create and (un)load</li>
204 <li>do not require extra privileges</li>
205 <li>don't repeat yourself (DRY)</li>
206 <li>easy to update without breaking the API</li>
207 <li>easy to debug</li>
208 <li>play nicely with the VCS (git)</li>
209 </ul>
210 </section>
211
212 <section>
213 <h3>Creating and maintaining a Python environment (cont...)</h3>
214 </br>
215 <p>Exploting the operating system can be done by:</p>
216 <ul>
217 <li>(re)defining <em>PYTHONPATH</em></li>
218 <li>using symlinks to point at packages placed at different locations</li>
219 </section>
220
221 <section>
222 <h3>Creating and maintaining a Python environment (cont...)</h3>
223 </br>
224 <p>Using <em>venv</em> can be done by directly invoking <em>python</em>:</p>
225 <pre data-id="code-animation">
226 <code class="bash" data-trim type="text/template">
227 # create a virtual environment
228 python -m venv my_virtual_env
229 python -m venv --system-site-packages my_virtual_env
230
231 # load, use and unload the virtual environment
232 source my_virtual_env/bin/activate
233 pip install sqlalchemy
234 # install package from a custom repository (https://artifactory.fifty.eu)
235 pip install --index-url=https://artifactory.fifty.eu/artifactory/api/pypi/pypi/simple/ odin-data-access
236 deactivate
237
238 # one can alternatively use the python "wrapper" of the virtual env
239 my_virtual_env/bin/python -m pip install sqlalchemy
240 </code>
241 </pre>
242 <p><em>--system-site-packages</em> will keep the original <em>site-packages</em> folders at the end of <em>sys.path</em></p>
243 </section>
244
245 <section>
246 <h3>Creating and maintaining a Python environment (cont...)</h3>
247 </br>
248 <p><em>Poetry</em> (<a href="https://python-poetry.org"><em>https://python-poetry.org</em></a>) is the prefered environment and dependency management tool at Statnett.</p>
249 <pre data-id="code-animation">
250 <code class="bash" data-trim type="text/template">
251 # create project and a virtual environment from scratch
252 poetry new my-project
253
254 # ... or use Poetry with an existing one
255 cd my-project
256 poetry init
257
258 # edit pyproject.toml for your needs (f.i. add dependencies, metadata ... etc),
259 # create virtual environment and install dependencies
260 poetry install
261 # finally commit your poetry.lock file to version control
262
263 # update all dependencies
264 poetry update
265 </code>
266 </pre>
267 <p>For more info: <a href="https://python-poetry.org/docs/basic-usage/"><em>https://python-poetry.org/docs/basic-usage/</em></a></p>
268 </section>
269
270 <section>
271 <h2>Mutables vs. immutables</h2>
272 <p>Immutable object is an object with a fixed value. Immutable objects include <em>bool, int, float, str, bytes</em> and <em>tuples</em>. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key in a dictionary.</p>
273 <p>All objects that are not immutable are... mutable. All <em>hashable objects</em> <b>should</b> be immutable or use <em>id()</em>.</p>
274 <pre data-id="code-animation">
275 <code class="python" data-trim type="text/template">
276 i = 1
277 id(i) # returns f.i. 9788992
278 i += 1 # same as i = i + 1
279 id(i) # returns a different value, hence - a brand new object
280
281 s = 'Hello'
282 s += ' World' # s is now a different object
283 s[0] # 'H'
284 s[0] = 'h' # TypeError: 'str' object does not support item assignment
285
286 t = (1, 4) # tuple
287 l = [1, 4] # list
288 hash(t) # returns f.i. -6333845781340707986
289 hash(l) # TypeError: unhashable type: 'list'
290
291 s = 'the long and winding road'
292 s2 = 'the long and winding road'
293
294 # check if s and s2 are the same object:
295 id(s) # Out: 139858905258704
296 id(s2) # Out: 139858926037472
297
298 # the hash should be the same
299 hash(s) # Out: 7030216208569256362
300 hash(s2) # Out: 7030216208569256362
301 </code>
302 </pre>
303 </section>
304
305 <section>
306 <h2>Functions</h2>
307 </br>
308 <p>A function is a sequence of program instructions that performs a specific task, packaged as a unit.</p>
309 <p>Functions let you:</p>
310 <ul>
311 <li>reuse code across several programs / projects</li>
312 <li>minimize code duplication</li>
313 <li>devide larger programming tasks</li>
314 <li>hide implementation details</li>
315 <li>improve readability</li>
316 <li>improve traceability</li>
317 </ul>
318 <p>Function calls bring some overhead pushing / popping function-data into / from stack.</p>
319 <p>Important definitions (may have different meanings in different programming languages):</p>
320 <ul>
321 <li><em>parameter / formal parameter</em> - variable / data provided as input to the function</li>
322 <li><em>argument / actual parameter</em> - local variable / data to the given function</li>
323 </ul>
324 <p>The keyword <em>def</em> introduces a function definition.</p>
325 </section>
326
327 <section>
328 <h2>Functions (cont...)</h2>
329 </br>
330 <pre data-id="code-animation">
331 <code class="python" data-trim type="text/template">
332 def add(a, b): # - function definition / header
333 """Function for adding integers""" # - docstring
334 result = a + b
335 a = 5
336 return result # - function that does not contain return, implicitly returns None
337
338 a_param = 9
339 b_param = -2
340 add(a_param, b_param) # Out: 7
341
342 # integers are immutable and a_param will remain unchanged
343 print(a_param) # Out: 9
344
345
346 def addl(a, b):
347 """Function for adding two lists"""
348 result = a + b
349 a += [5] # in this case equal to: a.append(5)
350 return result
351
352 a_param = [9]
353 b_param = [-2]
354 addl(a_param, b_param) # Out: [9, -2]
355 # lists are mutable and a_param will be changed
356 print(a_param) # Out: [9, 5]
357 </code>
358 </pre>
359 </section>
360
361 <section>
362 <h2>Functions (cont ...)</h2>
363 </br>
364 <h3>Parameters and arguments</h3>
365 <pre data-id="code-animation">
366 <code class="python" data-trim type="text/template">
367 def add(a, b):
368 """Function for adding integers"""
369 return a + b
370 my_result = add(2, 5) # positional arguments (parameters)
371 my_result = add(b=5, a=2) # keyword arguments (parameters)
372 my_tuple = (2, 5)
373 my_dict = {'b': 5, 'a': 2}
374 my_result = add(*my_tuple) # unpacked and assigned to the positional arguments
375 my_result = add(**my_dict) # unpacked and assigned to the kw. arguments
376
377 def add(a, b=5):
378 """Function for adding integers"""
379 return a + b
380 my_result = add(2)
381 # ... and the rest of the examples above will work
382
383 def add(a, *args, **kwargs):
384 """Function for adding integers"""
385 if args:
386 b = args[0]
387 elif 'b' in kwargs:
388 b = kwargs['b']
389 return a + b
390 my_result = add(2, 5, 9, 11) # 5 assigned to args[0]
391 my_result = add(2, b=5, c=9, d=11) # 5 assigned to kwargs['b']
392 my_result = add(2, b=5, 9, 11) # SyntaxError: positional argument follows keyword argument
393 </code>
394 </pre>
395 </section>
396
397 <section>
398 <h2>Functions (cont ...)</h2>
399 </br>
400 <h3>Docstrings annotations and other hints</h3>
401 <pre data-id="code-animation">
402 <code class="python" data-trim type="text/template">
403 def decrypt(password: str, edata: str) -> str:
404 """
405 Decrypts `edata` using `password`.
406
407 `edata` is in the following format:
408 enc-val$`version-num`$`bas64-salt`$`base64-encrypted_data`
409
410 :param password: The password to generate the key with
411 :type password: str
412
413 :param edata: The data to be decrypted
414 :type edata: str
415
416 :raises EtoolkitInstanceError: If the encryption format is unsupported
417
418 :return: The output string / decrypted data
419 :rtype: str
420 """
421 if not edata.startswith('enc-val$1$'):
422 raise EtoolkitInstanceError('Unsupported encryption format')
423 # some more code magic coming after....
424 # ...
425 # ..
426 return decrypted_str
427 </code>
428 </pre>
429 </section>
430
431 <section>
432 <h2>Functions (cont ...)</h2>
433 </br>
434 <h3>Using <em>typing</em> for more advanced / flexible hinting</h3>
435 <pre data-id="code-animation">
436 <code class="python" data-trim type="text/template">
437 import typing
438
439 Basestring = typing.Union[str, bytes]
440
441 def decrypt(password: Basestring, edata: str) -> str:
442 pass
443
444
445 from typing import Union
446
447 def decrypt(password: Union[str, bytes], edata: str) -> str:
448 """Generic documentation. No need for pass"""
449
450 # Python >= 3.10 only
451 def decrypt(password: str | bytes, edata: str) -> str:
452 """Generic documentation. No need for pass"""
453 </code>
454 </pre>
455 </section>
456
457 <section>
458 <h2>Functions (cont ...)</h2>
459 <h4>Functions as parameters / arguments, lambdas and returning multiple values</h4>
460 <pre data-id="code-animation">
461 <code class="python" data-trim type="text/template">
462 def fetch_the_first_letter(input_str: str) -> str:
463 """Fetches the first letter of the string input_str or 'x'""""
464 try:
465 return input_str[0]
466 except Exception:
467 return 'x'
468 letter_list = list(map(fetch_the_first_letter, ['foo', 'bar', 'test'])) # Out: ['f', 'b', 't']
469 </code>
470 </pre>
471 <p>Small anonymous functions can be created with the <em>lambda</em> keyword</p>
472 <pre data-id="code-animation">
473 <code class="python" data-trim type="text/template">
474 letter_list = list(map(lambda x: x[0], ['foo', 'bar', 'test'])) # Out: ['f', 'b', 't']
475 </code>
476 </pre>
477 <p>Functions in Python are <em>callable objects</em>. Callable objects can be created by defining the <em>__call__</em> method. More on that later in the course...</p>
478 <p>A function can return multiple values by implicitly returning a tuple:</p>
479 <pre data-id="code-animation">
480 <code class="python" data-trim type="text/template">
481 def square_cube(x):
482 """returns x, x^2 and x^3"""
483 return x, x**2, x**3
484 numbers = square_cube(5) # Out: (5, 25, 125)
485 num, sqnum, cbnum = square_cube(5) # unpacking the tuple
486 </code>
487 </pre>
488 </section>
489
490 <section>
491 <h2>Functions (cont ...)</h2>
492 </br>
493 <h3>Enclosing and nested functions</h3>
494 <p>Can be used as:</p>
495 <ul>
496 <li>regular functions within functions</li>
497 <li>dynamic function factories</li>
498 </ul>
499 <pre data-id="code-animation">
500 <code class="python" data-trim type="text/template">
501 def get_multiplier_of(base: int) -> str:
502 """the function enclosing its nested functions"""
503
504 def multiplier_function(x):
505 """a nested function"""
506 return base * x
507
508 return multiplier_function
509
510 times3 = get_multiplier_of(3)
511 times5 = get_multiplier_of(5)
512 print(times3(3)) # Out: 9
513 print(times5(3)) # Out: 15
514 </code>
515 </pre>
516 </section>
517
518 <section>
519 <h2>Scopes in Python</h2>
520 </br>
521 <ul>
522 <li><em>local</em> - assigned names are local unless declared <em>global</em></li>
523 <li><em>enclosed</em> - the scope of the variable inside a function with a nested function</li>
524 <li><em>global - global for the current module</em></li>
525 <li><em>built-in</em></li>
526 </ul>
527 <p><em>locals()</em> and <em>globals()</em> return dicts of symbols for their respective scopes</p>
528 <pre data-id="code-animation">
529 <code class="python" data-trim type="text/template">
530 num1, num2 = 7, 8 # module globals
531
532 def print_numbers():
533 print(num1, num2) # OK, these are module globals
534 num3 = 100
535 print(num3) # prints 100, num3 is in the function (local) scope
536 global num4 # assignes / references num4 to / in the global scope
537 num4 = 99
538 id = 200 # new symbol in local scope
539 # id(num4) # will not yield the expected result (raises TypeError)
540
541 def print_numbers2():
542 print(num3) # OK, enclosed scope
543
544 print_numbers2() # prints 100
545
546 print_numbers()
547 # print(num3) # Raises NameError - why?
548 print(num4) # Prints 99 - why?
549 # print_numbers2() # Raises NameError
550 </code>
551 </pre>
552 </section>
553
554 <section>
555 <h2>Decorators</h2>
556 <p>Decorators can be used to modify the behavior of the objects they decorate. Decorators can be implemented either by using classes or by using nested functions.</p>
557 <pre data-id="code-animation">
558 <code class="python" data-trim type="text/template">
559 def my_decorator(func):
560
561 def decorated():
562 print('Doing something before the decorated function')
563 retval = func()
564 print('Doing something after the decorated function')
565 return retval
566 return decorated
567
568 def my_function():
569 print('Alice')
570
571 my_function = my_decorator(my_function)
572 my_function()
573 </code>
574 </pre>
575 <p>... may be dificult to read / understand, while:</p>
576 <pre data-id="code-animation">
577 <code class="python" data-trim type="text/template">
578 @my_decorator
579 def my_function():
580 print('Alice')
581
582 my_function()
583 </code>
584 </pre>
585 <p>... may be easier</p>
586 </section>
587
588 <section>
589 <h2>Decorators (cont ...)</h2>
590 </br>
591 <h3>A complete example</h3>
592 <pre data-id="code-animation">
593 <code class="python" data-trim type="text/template">
594 import sys
595 from functools import wraps
596
597 def requires_access(access_secret: str):
598
599 def api_access_decorator(f):
600
601 @wraps(f)
602 def decorated(*args, **kwargs):
603 if 'secret' not in kwargs:
604 sys.exit('No secret provided')
605 if not kwargs['secret'] or kwargs['secret'] != access_secret:
606 sys.exit("Secret doesn't match")
607 # return f(args[0], **kwargs)
608 return f(*args, **kwargs)
609
610 return decorated
611
612 return api_access_decorator
613
614
615 @requires_access(access_secret='b28cfeaa65b73cf')
616 def sensitive_function(data, **kwargs):
617 """very sensitive function"""
618 db.save(data)
619 </code>
620 </pre>
621 </section>
622
623 <section>
624 <h2>String formatting</h2>
625 </br>
626 <h3>The old ways...</h3>
627 <pre data-id="code-animation">
628 <code class="python" data-trim type="text/template">
629 f = 6.57865
630 i = 27
631 s = 'another string'
632
633
634 '%s - %d - %5.2f' % (s, i, f) # Out: 'another string - 27 - 6.58'
635
636 # still used in:
637 logger.debug("%d - %s", event.id, message)
638
639
640 '{} - {} - {:5.2f}'.format(s, i, f) # implicit
641 '{0} - {1} - {2:5.2f}'.format(s, i, f) # explicit
642 '{my_str} - {i} - {fl:5.2f}'.format(my_str=s, fl=f, i=i) # keyword
643 # Out: 'another string - 27 - 6.58'
644
645
646 # modern Python >= 3.6 f-strings
647 f'{s} - {i} - {f:5.2f}' # Out: 'another string - 27 - 6.58'
648 </code>
649 </pre>
650 <p>See <a href="https://docs.python.org/3/library/string.html#formatspec">https://docs.python.org/3/library/string.html#formatspec</a> for the complete format specification</p>
651 </section>
652
653 </div>
654 </div>
655
656 <script src="dist/reveal.js"></script>
657 <script src="plugin/zoom/zoom.js"></script>
658 <script src="plugin/notes/notes.js"></script>
659 <script src="plugin/search/search.js"></script>
660 <script src="plugin/markdown/markdown.js"></script>
661 <script src="plugin/highlight/highlight.js"></script>
662 <script>
663
664 // Also available as an ES module, see:
665 // https://revealjs.netlify.app/initialization/
666 Reveal.initialize({
667 controls: true,
668 progress: true,
669 center: true,
670 hash: true,
671
672 // Learn about plugins: https://revealjs.netlify.app/plugins/
673 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
674 });
675 Reveal.configure({ pdfSeparateFragments: false });
676
677 </script>
678
679 </body>
680</html>
diff --git a/reveal.js/rabbitmq.html b/reveal.js/rabbitmq.html
index ed1cbfd..6d1ba7a 100644
--- a/reveal.js/rabbitmq.html
+++ b/reveal.js/rabbitmq.html
@@ -1,311 +1,311 @@
1<!doctype html> 1<!doctype html>
2<html lang="en"> 2<html lang="en">
3 <head> 3 <head>
4 <meta charset="utf-8"> 4 <meta charset="utf-8">
5 <title>RabbitMQ - Intern opplæring</title> 5 <title>RabbitMQ - Intern opplæring</title>
6 <meta name="author" content="Simeon Simeonov"> 6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes"> 7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"> 8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0"> 9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 10
11 <link rel="stylesheet" href="dist/reset.css"> 11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css"> 12 <link rel="stylesheet" href="dist/reveal.css">
13 13
14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme"> 14 <link rel="stylesheet" href="dist/theme/serif.css" id="theme">
15 15
16 <!-- Theme used for syntax highlighting of code --> 16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme"> 17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 </head> 18 </head>
19 <body> 19 <body>
20 <div class="reveal"> 20 <div class="reveal">
21 21
22 <!-- Any section element inside of this container is displayed as a slide --> 22 <!-- Any section element inside of this container is displayed as a slide -->
23 <div class="slides"> 23 <div class="slides">
24 <section> 24 <section>
25 <h1>RabbitMQ</h1> 25 <h1>RabbitMQ</h1>
26 <h3>Intern opplæring</h3> 26 <h3>Intern opplæring</h3>
27 </br> 27 </br>
28 <p> 28 <p>
29 <small>Simeon Simeonov</small> 29 <small>Simeon Simeonov</small>
30 </p> 30 </p>
31 </section> 31 </section>
32 32
33 <section> 33 <section>
34 <section id="fragments"> 34 <section id="fragments">
35 <h2>Agenda</h2> 35 <h2>Agenda</h2>
36 </br> 36 </br>
37 <ul> 37 <ul>
38 <span class="fragment"><li>Installasjon</li></span> 38 <span class="fragment"><li>Installasjon</li></span>
39 <span class="fragment"><li>Administrasjon</li></span> 39 <span class="fragment"><li>Administrasjon</li></span>
40 <span class="fragment"><li>Utvikling</li></span> 40 <span class="fragment"><li>Utvikling</li></span>
41 <span class="fragment"><li>Diverse + Q&A</li></span> 41 <span class="fragment"><li>Diverse + Q&A</li></span>
42 </ul> 42 </ul>
43 </section> 43 </section>
44 </section> 44 </section>
45 45
46 <section> 46 <section>
47 <section> 47 <section>
48 <h2>Installasjon og oppsett</h2> 48 <h2>Installasjon og oppsett</h2>
49 </br> 49 </br>
50 <ul> 50 <ul>
51 <li><p>Skaffe seg maskin og SSL sertifikat(er)</p></li> 51 <li><p>Skaffe seg maskin og SSL sertifikat(er)</p></li>
52 <li><p>Bruke Ansible</p></li> 52 <li><p>Bruke Ansible</p></li>
53 <li><p>Opprette administrator-bruker og slette guest-brukeren</p></li> 53 <li><p>Opprette administrator-bruker og slette guest-brukeren</p></li>
54 <li><p>Sette rettigheter og opprette entiteter</p></li> 54 <li><p>Sette rettigheter og opprette entiteter</p></li>
55 </ul> 55 </ul>
56 </section> 56 </section>
57 <section> 57 <section>
58 <h2>Ansible</h2> 58 <h2>Ansible</h2>
59 </br> 59 </br>
60 <p>Vi bruker repo <em>UAIT/int-ansible-hosts</em></p> 60 <p>Vi bruker repo <em>UAIT/int-ansible-hosts</em></p>
61 <p>Etter å ha installert Ansible på lokalmaskina gjør vi følgende:</p> 61 <p>Etter å ha installert Ansible på lokalmaskina gjør vi følgende:</p>
62 <ul> 62 <ul>
63 <li><p><small><em>git clone ssh://git@bitbucket.usit.uio.no:7999/uait/int-ansible-hosts.git</em></small></p></li> 63 <li><p><small><em>git clone ssh://git@bitbucket.usit.uio.no:7999/uait/int-ansible-hosts.git</em></small></p></li>
64 <li><p><small><em>cd int-ansible-hosts</em></small></p></li> 64 <li><p><small><em>cd int-ansible-hosts</em></small></p></li>
65 <li><p><small><em>ansible-playbook --ask-become-pass -v --extra-vars '{"hosts": "mq-hostname"}' rabbitmq_playbook.yml</em></small></p></li> 65 <li><p><small><em>ansible-playbook --ask-become-pass -v --extra-vars '{"hosts": "mq-hostname"}' rabbitmq_playbook.yml</em></small></p></li>
66 </ul> 66 </ul>
67 <p>RabbitMQ skal nå være installert og tilgjengelig på <em>https://mq-hostname</em></p> 67 <p>RabbitMQ skal nå være installert og tilgjengelig på <em>https://mq-hostname</em></p>
68 </section> 68 </section>
69 69
70 <section> 70 <section>
71 <h2>Administrator-bruker</h2> 71 <h2>Administrator-bruker</h2>
72 </br> 72 </br>
73 <p>Brukeren <em>guest</em> med passord <em>guest</em> vil eksistere etter at Ansible har kjørt</p> 73 <p>Brukeren <em>guest</em> med passord <em>guest</em> vil eksistere etter at Ansible har kjørt</p>
74 <p>I tillegg til webgrensesnittet, kan en bruke <em>rabbitmqctl</em></p> 74 <p>I tillegg til webgrensesnittet, kan en bruke <em>rabbitmqctl</em></p>
75 <ul> 75 <ul>
76 <li><p><small><em>rabbitmqctl list_users</em> (Lister alle eksisterende RabbitMQ-brukere)</small></p></li> 76 <li><p><small><em>rabbitmqctl list_users</em> (Lister alle eksisterende RabbitMQ-brukere)</small></p></li>
77 <li><p><small><em>rabbitmqctl add_user rmq_admin &lt;passord&gt;</em> (Legger til dedikert admin bruker <em>rmq_admin</em>)</small></p></li> 77 <li><p><small><em>rabbitmqctl add_user rmq_admin &lt;passord&gt;</em> (Legger til dedikert admin bruker <em>rmq_admin</em>)</small></p></li>
78 <li><p><small><em>rabbitmqctl set_user_tags rmq_admin administrator</em> (Gir administrator rolle til <em>rmq_admin</em>)</small></p></li> 78 <li><p><small><em>rabbitmqctl set_user_tags rmq_admin administrator</em> (Gir administrator rolle til <em>rmq_admin</em>)</small></p></li>
79 <li><p><small><em>rabbitmqctl delete_user guest</em> (Sletter brukeren <em>guest</em>)</small></p></li> 79 <li><p><small><em>rabbitmqctl delete_user guest</em> (Sletter brukeren <em>guest</em>)</small></p></li>
80 </ul> 80 </ul>
81 </section> 81 </section>
82 82
83 <section> 83 <section>
84 <h2><em>vhost</em></h2> 84 <h2><em>vhost</em></h2>
85 </br> 85 </br>
86 <p><em>vhost</em> fungerer som en beholder (container) for RabbitMQ/AMQP objekter. Nok igjen kan en velge mellom webgrensesnitt og <em>rabbitmqctl</em></p> 86 <p><em>vhost</em> fungerer som en beholder (container) for RabbitMQ/AMQP objekter. Nok igjen kan en velge mellom webgrensesnitt og <em>rabbitmqctl</em></p>
87 <ul> 87 <ul>
88 <li><p><small><em>rabbitmqctl list_vhosts</em> (Lister alle eksisterende vhosts)</small></p></li> 88 <li><p><small><em>rabbitmqctl list_vhosts</em> (Lister alle eksisterende vhosts)</small></p></li>
89 <li><p><small><em>rabbitmqctl add_vhost /no/uio/integration</em> (Oppretter <em>vhost</em> med navn <em>/no/uio/integration</em>)</small></p></li> 89 <li><p><small><em>rabbitmqctl add_vhost /no/uio/integration</em> (Oppretter <em>vhost</em> med navn <em>/no/uio/integration</em>)</small></p></li>
90 </ul> 90 </ul>
91 </section> 91 </section>
92 92
93 <section> 93 <section>
94 <h2>Rettigheter</h2> 94 <h2>Rettigheter</h2>
95 <p>RabbitMQ implementerer 2 rettighetsnivåer(*):</p> 95 <p>RabbitMQ implementerer 2 rettighetsnivåer(*):</p>
96 <ul> 96 <ul>
97 <li>Per <em>vhost</em></li> 97 <li>Per <em>vhost</em></li>
98 <li>Per entitet</li> 98 <li>Per entitet</li>
99 </ul> 99 </ul>
100 <p>RabbitMQ definerer 3 typer operasjoner:</p> 100 <p>RabbitMQ definerer 3 typer operasjoner:</p>
101 <ul> 101 <ul>
102 <li><p><em>configure</em> - opprette / slette entiteter eller endre deres oppførsel</p></li> 102 <li><p><em>configure</em> - opprette / slette entiteter eller endre deres oppførsel</p></li>
103 <li><p><em>write</em> - skrive melding til en entitet</p></li> 103 <li><p><em>write</em> - skrive melding til en entitet</p></li>
104 <li><p><em>read</em> - lese melding fra entitet</p></li> 104 <li><p><em>read</em> - lese melding fra entitet</p></li>
105 </ul> 105 </ul>
106 <p><small>(*) - <em>/etc/rabbitmq/rabbitmq.config</em> gir flere muligheter</small></p> 106 <p><small>(*) - <em>/etc/rabbitmq/rabbitmq.config</em> gir flere muligheter</small></p>
107 </section> 107 </section>
108 108
109 <section> 109 <section>
110 <h2>Rettigheter og brukere</h2> 110 <h2>Rettigheter og brukere</h2>
111 <p>RabbitMQ bruker <em>regular expressions</em> til å definere rettigheter</p> 111 <p>RabbitMQ bruker <em>regular expressions</em> til å definere rettigheter</p>
112 <ul> 112 <ul>
113 <li><p><small><em>rabbitmqctl add_user cerebrum &lt;passord&gt;</em></small></p></li> 113 <li><p><small><em>rabbitmqctl add_user cerebrum &lt;passord&gt;</em></small></p></li>
114 <li><p><small><em>rabbitmqctl add_user uio_ad_microservice &lt;passord&gt;</em></small></p></li> 114 <li><p><small><em>rabbitmqctl add_user uio_ad_microservice &lt;passord&gt;</em></small></p></li>
115 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"</em></small></p></li> 115 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"</em></small></p></li>
116 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"</em></small></p></li> 116 <li><p><small><em>rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"</em></small></p></li>
117 </ul> 117 </ul>
118 </section> 118 </section>
119 119
120 <section> 120 <section>
121 <h2>Køer</h2> 121 <h2>Køer</h2>
122 <p>Køer defineres som <em>durable</em> ved hjelp av webgrensesnittet.</p> 122 <p>Køer defineres som <em>durable</em> ved hjelp av webgrensesnittet.</p>
123 <p>"Alle køer (både forhåndsdefinerte og de som defineres av konsument) må ha navn som starter med <em>q_</em>. Køen vil ha navn som uttryker mottakeren som bruker den."</p> 123 <p>"Alle køer (både forhåndsdefinerte og de som defineres av konsument) må ha navn som starter med <em>q_</em>. Køen vil ha navn som uttryker mottakeren som bruker den."</p>
124 <p>F.eks. <em>q_ad_ms_all</em> vil være et passende navn for køen som brukes av AD-microservice mottakeren.</p> 124 <p>F.eks. <em>q_ad_ms_all</em> vil være et passende navn for køen som brukes av AD-microservice mottakeren.</p>
125 </section> 125 </section>
126 126
127 <section> 127 <section>
128 <h2><em>Exchange</em></h2> 128 <h2><em>Exchange</em></h2>
129 <p>"Avsendere vil vanligvis sende meldinger til exchange <em>ex_messages</em>."</p> 129 <p>"Avsendere vil vanligvis sende meldinger til exchange <em>ex_messages</em>."</p>
130 <p>Det er flere typer <em>exchange</em>, men vi vil bruke kun <em>topic exchange</em>.</p> 130 <p>Det er flere typer <em>exchange</em>, men vi vil bruke kun <em>topic exchange</em>.</p>
131 </section> 131 </section>
132 </section> 132 </section>
133 133
134 <section> 134 <section>
135 <section> 135 <section>
136 <h2>Utvikling</h2> 136 <h2>Utvikling</h2>
137 </br> 137 </br>
138 <ul> 138 <ul>
139 <li><p><em>Routing keys</em> i <em>topic exchange</em></p></li> 139 <li><p><em>Routing keys</em> i <em>topic exchange</em></p></li>
140 <li><p>Bindinger</p></li> 140 <li><p>Bindinger</p></li>
141 <li><p>Protokoller og porter</p></li> 141 <li><p>Protokoller og porter</p></li>
142 <li><p>Eksempler</p></li> 142 <li><p>Eksempler</p></li>
143 </ul> 143 </ul>
144 </section> 144 </section>
145 145
146 <section> 146 <section>
147 <h2><em>Routing keys</em> i <em>topic exchange</em></h2> 147 <h2><em>Routing keys</em> i <em>topic exchange</em></h2>
148 </br> 148 </br>
149 <p><em>Routing key</em> settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en <em>topic exchange</em>.</p> 149 <p><em>Routing key</em> settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en <em>topic exchange</em>.</p>
150 <p>Strukturen til en topic / message routing key er:</p> 150 <p>Strukturen til en topic / message routing key er:</p>
151 <p><small>&lt;kilde&gt;.&lt;type&gt;.&lt;objekt&gt;.&lt;hendelse&gt;</small></p> 151 <p><small>&lt;kilde&gt;.&lt;type&gt;.&lt;objekt&gt;.&lt;hendelse&gt;</small></p>
152 <p>F.eks.:</p> 152 <p>F.eks.:</p>
153 <p><small>cerebrum.event.person.delete</small></p> 153 <p><small>cerebrum.event.person.delete</small></p>
154 </section> 154 </section>
155 155
156 <section> 156 <section>
157 <h2>Bindinger</h2> 157 <h2>Bindinger</h2>
158 <p>For at en melding sendt til en <em>topic exchange</em> skal havne i en bestemt kø, må køen være bundet (<em>bound</em>) til <em>exchange</em>.</p> 158 <p>For at en melding sendt til en <em>topic exchange</em> skal havne i en bestemt kø, må køen være bundet (<em>bound</em>) til <em>exchange</em>.</p>
159 <p>I <em>topic exchange</em> brukes <em>routing key</em> (<em>topic</em>) til å avgjøre hvilke av meldingene som blir sendt vil havne i køen som er bundet.</p> 159 <p>I <em>topic exchange</em> brukes <em>routing key</em> (<em>topic</em>) til å avgjøre hvilke av meldingene som blir sendt vil havne i køen som er bundet.</p>
160 <p>Dersom vi binder køen <em>q_ad_ms_all</em> til <em>topic exchange</em> <em>ex_messages</em> med <em>binding key</em> <em>cerebrum.event.account.*</em> vil alle meldinger som blir sendt til <em>ex_messages</em> med topic som starter med <em>cerebrum.event.account.</em> havne i <em>q_ad_ms_all</em>.</p> 160 <p>Dersom vi binder køen <em>q_ad_ms_all</em> til <em>topic exchange</em> <em>ex_messages</em> med <em>binding key</em> <em>cerebrum.event.account.*</em> vil alle meldinger som blir sendt til <em>ex_messages</em> med topic som starter med <em>cerebrum.event.account.</em> havne i <em>q_ad_ms_all</em>.</p>
161 <p>En kø kan bli bundet med én eller flere nøkler til en <em>topic exchange</em>.</p> 161 <p>En kø kan bli bundet med én eller flere nøkler til en <em>topic exchange</em>.</p>
162 </section> 162 </section>
163 163
164 <section> 164 <section>
165 <h2>Protokoller og porter</h2> 165 <h2>Protokoller og porter</h2>
166 </br> 166 </br>
167 <p>"AMQP 0.9.1 prioritert siden det er protokollen som mapper best mot RabbitMQs funksjonalitet. Vi har også noe erfaring med bruk av denne."</p> 167 <p>"AMQP 0.9.1 prioritert siden det er protokollen som mapper best mot RabbitMQs funksjonalitet. Vi har også noe erfaring med bruk av denne."</p>
168 <p>RabbitMQ lytter for AMQP0.9.1 på SSL port <em>5671</em>.</p> 168 <p>RabbitMQ lytter for AMQP0.9.1 på SSL port <em>5671</em>.</p>
169 </br> 169 </br>
170 <p>Andre protokoller og tjenester for RabbitMQ:</p> 170 <p>Andre protokoller og tjenester for RabbitMQ:</p>
171 <ul> 171 <ul>
172 <li>STOMP - 61614 (SSL)</li> 172 <li>STOMP - 61614 (SSL)</li>
173 <li>RabbitMQ Management (webgrensesnitt) - 15671 (SSL)</li> 173 <li>RabbitMQ Management (webgrensesnitt) - 15671 (SSL)</li>
174 </ul> 174 </ul>
175 </section> 175 </section>
176 176
177 <section> 177 <section>
178 <h2>Eksempler</h2> 178 <h2>Eksempler</h2>
179 </br> 179 </br>
180 <pre> 180 <pre>
181 <code class="hljs" data-trim contenteditable> 181 <code class="hljs" data-trim contenteditable>
182 $ pip install pika 182 $ pip install pika
183 </code> 183 </code>
184 </pre> 184 </pre>
185 <p>Python:</p> 185 <p>Python:</p>
186 <p>"Well, we'll not risk another frontal assault. That rabbit's dynamite...."</p> 186 <p>"Well, we'll not risk another frontal assault. That rabbit's dynamite...."</p>
187 </section> 187 </section>
188 188
189 </section> 189 </section>
190 190
191 <section> 191 <section>
192 <section> 192 <section>
193 <h2>Dokumentasjon og lenker</h2> 193 <h2>Dokumentasjon og lenker</h2>
194 </br> 194 </br>
195 <p> 195 <p>
196 RabbitMQ: 196 RabbitMQ:
197 </p> 197 </p>
198 <ul> 198 <ul>
199 <li> 199 <li>
200 <p> 200 <p>
201 <small> 201 <small>
202 <a href="http://www.rabbitmq.com">http://www.rabbitmq.com</a> - RabbitMQ sin offisielle side 202 <a href="http://www.rabbitmq.com">http://www.rabbitmq.com</a> - RabbitMQ sin offisielle side
203 </small> 203 </small>
204 </li> 204 </li>
205 <li> 205 <li>
206 <p> 206 <p>
207 <small> 207 <small>
208 <a href="http://www.rabbitmq.com/configure.html">http://www.rabbitmq.com/configure.html</a> - Generell konfigurasjon av RabbitMQ 208 <a href="http://www.rabbitmq.com/configure.html">http://www.rabbitmq.com/configure.html</a> - Generell konfigurasjon av RabbitMQ
209 </small> 209 </small>
210 </li> 210 </li>
211 <li> 211 <li>
212 <p> 212 <p>
213 <small> 213 <small>
214 <a href="http://www.rabbitmq.com/management.html">http://www.rabbitmq.com/management.html</a> - Bruk av rabbitmq_management plugin i RabbitMQ 214 <a href="http://www.rabbitmq.com/management.html">http://www.rabbitmq.com/management.html</a> - Bruk av rabbitmq_management plugin i RabbitMQ
215 </small> 215 </small>
216 </li> 216 </li>
217 <li> 217 <li>
218 <p> 218 <p>
219 <small> 219 <small>
220 <a href="http://www.rabbitmq.com/access-control.html">http://www.rabbitmq.com/access-control.html</a> - Tilgangskontroll i RabbitMQ 220 <a href="http://www.rabbitmq.com/access-control.html">http://www.rabbitmq.com/access-control.html</a> - Tilgangskontroll i RabbitMQ
221 </small> 221 </small>
222 </li> 222 </li>
223 <li> 223 <li>
224 <p> 224 <p>
225 <small> 225 <small>
226 <a href="http://www.rabbitmq.com/man/rabbitmqctl.1.man.html">http://www.rabbitmq.com/man/rabbitmqctl.1.man.html</a> - rabbitmqctl manual side 226 <a href="http://www.rabbitmq.com/man/rabbitmqctl.1.man.html">http://www.rabbitmq.com/man/rabbitmqctl.1.man.html</a> - rabbitmqctl manual side
227 </small> 227 </small>
228 </li> 228 </li>
229 <li> 229 <li>
230 <p> 230 <p>
231 <small> 231 <small>
232 <a href="http://www.rabbitmq.com/tutorials/amqp-concepts.html">http://www.rabbitmq.com/tutorials/amqp-concepts.html</a> - Generelt om AMQP og forskjellige exchanges 232 <a href="http://www.rabbitmq.com/tutorials/amqp-concepts.html">http://www.rabbitmq.com/tutorials/amqp-concepts.html</a> - Generelt om AMQP og forskjellige exchanges
233 </small> 233 </small>
234 </li> 234 </li>
235 </ul> 235 </ul>
236 </section> 236 </section>
237 <section> 237 <section>
238 <h2>Dokumentasjon og lenker</h2> 238 <h2>Dokumentasjon og lenker</h2>
239 <p> 239 <p>
240 UiO: 240 UiO:
241 </p> 241 </p>
242 <ul> 242 <ul>
243 <li> 243 <li>
244 <p> 244 <p>
245 <small> 245 <small>
246 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/dokumentasjon/tools/rabbitmq.html">Installasjon og oppsett av RabbitMQ for Cerebrum utviklere</a> 246 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/dokumentasjon/tools/rabbitmq.html">Installasjon og oppsett av RabbitMQ for Cerebrum utviklere</a>
247 </small> 247 </small>
248 </li> 248 </li>
249 <li> 249 <li>
250 <p> 250 <p>
251 <small> 251 <small>
252 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/aktiviteter/office365-uia/broker/rabbitmq.html">Oppsett av RabbitMQ for Office365 integrasjon mot UiA</a> 252 <a href="http://www.usit.uio.no/om/tjenestegrupper/cerebrum/utvikling/aktiviteter/office365-uia/broker/rabbitmq.html">Oppsett av RabbitMQ for Office365 integrasjon mot UiA</a>
253 </small> 253 </small>
254 </li> 254 </li>
255 <li> 255 <li>
256 <p> 256 <p>
257 <small> 257 <small>
258 <a href="https://www.usit.uio.no/om/organisasjon/uav/usitint/interndokumentasjon/integrasjon/broker/utredning-meldingsflyt.html">Utredning: Meldingsflyt, kø-struktur og navngiving</a> (intern) 258 <a href="https://www.usit.uio.no/om/organisasjon/uav/usitint/interndokumentasjon/integrasjon/broker/utredning-meldingsflyt.html">Utredning: Meldingsflyt, kø-struktur og navngiving</a> (intern)
259 </small> 259 </small>
260 </li> 260 </li>
261 </ul> 261 </ul>
262 <p> 262 <p>
263 Bøker: 263 Bøker:
264 </p> 264 </p>
265 <ul> 265 <ul>
266 <li> 266 <li>
267 <p> 267 <p>
268 <small> 268 <small>
269 RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning 269 RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning
270 </small> 270 </small>
271 </li> 271 </li>
272 <li> 272 <li>
273 <p> 273 <p>
274 <small> 274 <small>
275 Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing 275 Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing
276 </small> 276 </small>
277 </li> 277 </li>
278 </section> 278 </section>
279 </section> 279 </section>
280
281 <section>
282 <h1>Q&A</h1>
283 </section>
284
285 </div>
286 </div>
280 287
281 <section> 288 <script src="dist/reveal.js"></script>
282 <h1>Q&A</h1> 289 <script src="plugin/zoom/zoom.js"></script>
283 </section> 290 <script src="plugin/notes/notes.js"></script>
291 <script src="plugin/search/search.js"></script>
292 <script src="plugin/markdown/markdown.js"></script>
293 <script src="plugin/highlight/highlight.js"></script>
294 <script>
284 295
285 </div> 296 // Also available as an ES module, see:
286 </div> 297 // https://revealjs.com/initialization/
287 298 Reveal.initialize({
288 <script src="dist/reveal.js"></script> 299 controls: true,
289 <script src="plugin/zoom/zoom.js"></script> 300 progress: true,
290 <script src="plugin/notes/notes.js"></script> 301 center: true,
291 <script src="plugin/search/search.js"></script> 302 hash: true,
292 <script src="plugin/markdown/markdown.js"></script>
293 <script src="plugin/highlight/highlight.js"></script>
294 <script>
295 303
296 // Also available as an ES module, see: 304 // Learn about plugins: https://revealjs.com/plugins/
297 // https://revealjs.netlify.app/initialization/ 305 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
298 Reveal.initialize({ 306 });
299 controls: true,
300 progress: true,
301 center: true,
302 hash: true,
303 307
304 // Learn about plugins: https://revealjs.netlify.app/plugins/ 308 </script>
305 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
306 });
307
308 </script>
309 309
310 </body> 310 </body>
311</html> 311</html>
diff --git a/reveal.js/sqlalchemy.html b/reveal.js/sqlalchemy.html
index a0d6617..bbcd1ca 100755..100644
--- a/reveal.js/sqlalchemy.html
+++ b/reveal.js/sqlalchemy.html
@@ -98,29 +98,29 @@
98 metadata = MetaData() 98 metadata = MetaData()
99 99
100 production_types_table = Table( 100 production_types_table = Table(
101 "production_types", 101 "production_types",
102 metadata, 102 metadata,
103 Column("production_type_id", Integer, primary_key=True), 103 Column("production_type_id", Integer, primary_key=True),
104 Column("code", String(3), nullable=False, unique=True), 104 Column("code", String(3), nullable=False, unique=True),
105 Column("description", String), # Column("name", String(128)) is possible 105 Column("description", String), # Column("name", String(128)) is possible
106 ) 106 )
107 107
108 bidding_areas_table = Table( 108 bidding_areas_table = Table(
109 "bidding_areas", 109 "bidding_areas",
110 metadata, 110 metadata,
111 Column("bidding_area_id", Integer, primary_key=True), 111 Column("bidding_area_id", Integer, primary_key=True),
112 Column("code", String(3), nullable=False, unique=True), 112 Column("code", String(3), nullable=False, unique=True),
113 Column("name", String(32)), 113 Column("name", String(32)),
114 ) 114 )
115 115
116 production_plans_table = Table( 116 production_plans_table = Table(
117 "production_plans", 117 "production_plans",
118 metadata, 118 metadata,
119 Column("record_created_time", DateTime(timezone=False), primary_key=True), 119 Column("record_created_time", DateTime(timezone=False), primary_key=True),
120 Column("start_time", DateTime(timezone=False), primary_key=True), 120 Column("start_time", DateTime(timezone=False), primary_key=True),
121 Column("bidding_area_id", Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True), 121 Column("bidding_area_id", Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True),
122 Column("production_type_id", Integer, ForeignKey("production_types.production_type_id"), primary_key=True), 122 Column("production_type_id", Integer, ForeignKey("production_types.production_type_id"), primary_key=True),
123 Column("value", Numeric, nullable=False), 123 Column("value", Numeric, nullable=False),
124 ) 124 )
125 125
126 metadata.create_all(engine) # creates the tables 126 metadata.create_all(engine) # creates the tables
@@ -150,13 +150,13 @@
150 insert_stmt.execute(bidding_area_id=1, code="NO1", name="Elspot NO1") # insert a single entry 150 insert_stmt.execute(bidding_area_id=1, code="NO1", name="Elspot NO1") # insert a single entry
151 # ... or a list of entries 151 # ... or a list of entries
152 insert_stmt.execute( 152 insert_stmt.execute(
153 [ 153 [
154 {"bidding_area_id": 2, "code": "NO2", "name": "Elspot NO2"}, 154 {"bidding_area_id": 2, "code": "NO2", "name": "Elspot NO2"},
155 {"bidding_area_id": 3, "code": "NO3", "name": "Elspot NO3"}, 155 {"bidding_area_id": 3, "code": "NO3", "name": "Elspot NO3"},
156 {"bidding_area_id": 4, "code": "NO4", "name": "Elspot NO4"}, 156 {"bidding_area_id": 4, "code": "NO4", "name": "Elspot NO4"},
157 {"bidding_area_id": 5, "code": "NO5", "name": "Elspot NO5"}, 157 {"bidding_area_id": 5, "code": "NO5", "name": "Elspot NO5"},
158 {"bidding_area_id": 6, "code": "NO6", "name": "Elspot NO6"}, 158 {"bidding_area_id": 6, "code": "NO6", "name": "Elspot NO6"},
159 ] 159 ]
160 ) 160 )
161 161
162 metadata.bind = engine # no need to explicitly bind the engine from now on 162 metadata.bind = engine # no need to explicitly bind the engine from now on
@@ -182,21 +182,21 @@
182 from sqlalchemy.orm import mapper 182 from sqlalchemy.orm import mapper
183 183
184 class ProductionType: 184 class ProductionType:
185 def __init__(self, code, description): 185 def __init__(self, code, description):
186 self.code = code 186 self.code = code
187 self.description = description 187 self.description = description
188 188
189 def __str__(self): 189 def __str__(self):
190 return self.code 190 return self.code
191 191
192 192
193 class BiddingArea: 193 class BiddingArea:
194 def __init__(self, code, name): 194 def __init__(self, code, name):
195 self.code = code 195 self.code = code
196 self.name = name 196 self.name = name
197 197
198 def __str__(self): 198 def __str__(self):
199 return self.code 199 return self.code
200 200
201 mapper(ProductionType, production_types_table) 201 mapper(ProductionType, production_types_table)
202 mapper(BiddingArea, bidding_areas_table) 202 mapper(BiddingArea, bidding_areas_table)
@@ -212,27 +212,27 @@
212 from sqlalchemy.orm import relationship 212 from sqlalchemy.orm import relationship
213 213
214 class ProductionPlan: 214 class ProductionPlan:
215 def __init__(self, record_created_time, start_time, production_type, bidding_area, value): 215 def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
216 self.record_created_time = record_created_time 216 self.record_created_time = record_created_time
217 self.start_time = start_time 217 self.start_time = start_time
218 self.production_type = production_type 218 self.production_type = production_type
219 self.bidding_area = bidding_area 219 self.bidding_area = bidding_area
220 self.value = value 220 self.value = value
221 221
222 def __str__(self): 222 def __str__(self):
223 return ( 223 return (
224 f"{self.record_created_time} {self.start_time} " 224 f"{self.record_created_time} {self.start_time} "
225 f"{self.production_type} {self.bidding_area} {self.value}" 225 f"{self.production_type} {self.bidding_area} {self.value}"
226 ) 226 )
227 227
228 228
229 mapper( 229 mapper(
230 ProductionPlan, 230 ProductionPlan,
231 production_plans_table, 231 production_plans_table,
232 properties = { 232 properties = {
233 "production_type": relationship(ProductionType, backref="production_plans"), 233 "production_type": relationship(ProductionType, backref="production_plans"),
234 "bidding_area": relationship(BiddingArea, backref="production_plans"), 234 "bidding_area": relationship(BiddingArea, backref="production_plans"),
235 }, 235 },
236 ) 236 )
237 </code> 237 </code>
238 </pre> 238 </pre>
@@ -249,32 +249,32 @@
249 Base = declarative_base() 249 Base = declarative_base()
250 250
251 class ProductionType(Base): 251 class ProductionType(Base):
252 __tablename__ = "production_types" 252 __tablename__ = "production_types"
253 253
254 production_type_id = Column(Integer, primary_key=True) 254 production_type_id = Column(Integer, primary_key=True)
255 code = Column(String(3), nullable=False, unique=True) 255 code = Column(String(3), nullable=False, unique=True)
256 description = Column(String) 256 description = Column(String)
257 257
258 def __init__(self, code, description): 258 def __init__(self, code, description):
259 self.code = code 259 self.code = code
260 self.description = description 260 self.description = description
261 261
262 def __str__(self): 262 def __str__(self):
263 return self.code 263 return self.code
264 264
265 class BiddingArea(Base): 265 class BiddingArea(Base):
266 __tablename__ = "bidding_areas" 266 __tablename__ = "bidding_areas"
267 267
268 bidding_area_id = Column(Integer, primary_key=True) 268 bidding_area_id = Column(Integer, primary_key=True)
269 code = Column(String(3), nullable=False, unique=True) 269 code = Column(String(3), nullable=False, unique=True)
270 name = Column(String(32)) 270 name = Column(String(32))
271 271
272 def __init__(self, code, name): 272 def __init__(self, code, name):
273 self.code = code 273 self.code = code
274 self.name = name 274 self.name = name
275 275
276 def __str__(self): 276 def __str__(self):
277 return self.code 277 return self.code
278 </code> 278 </code>
279 </pre> 279 </pre>
280 </section> 280 </section>
@@ -288,31 +288,31 @@
288 from sqlalchemy.orm import relationship, backref 288 from sqlalchemy.orm import relationship, backref
289 289
290 class ProductionPlan(Base): 290 class ProductionPlan(Base):
291 __tablename__ = "production_plans" 291 __tablename__ = "production_plans"
292 292
293 record_created_time = Column(DateTime(timezone=False), primary_key=True) 293 record_created_time = Column(DateTime(timezone=False), primary_key=True)
294 start_time = Column(DateTime(timezone=False), primary_key=True) 294 start_time = Column(DateTime(timezone=False), primary_key=True)
295 bidding_area_id = Column(Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True) 295 bidding_area_id = Column(Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True)
296 production_type_id = Column(Integer, ForeignKey("production_types.production_type_id"), primary_key=True) 296 production_type_id = Column(Integer, ForeignKey("production_types.production_type_id"), primary_key=True)
297 value = Column(Numeric, nullable=False) 297 value = Column(Numeric, nullable=False)
298 298
299 # defining relationships. 299 # defining relationships.
300 # the defined attributes will reference 'ProductionType' and 'BiddingArea' objects 300 # the defined attributes will reference 'ProductionType' and 'BiddingArea' objects
301 production_type = relationship(ProductionType, backref=backref("production_plans")) 301 production_type = relationship(ProductionType, backref=backref("production_plans"))
302 bidding_area = relationship(BiddingArea, backref=backref("production_plans")) 302 bidding_area = relationship(BiddingArea, backref=backref("production_plans"))
303 303
304 def __init__(self, record_created_time, start_time, production_type, bidding_area, value): 304 def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
305 self.record_created_time = record_created_time 305 self.record_created_time = record_created_time
306 self.start_time = start_time 306 self.start_time = start_time
307 self.production_type = production_type # a 'ProductionType' object 307 self.production_type = production_type # a 'ProductionType' object
308 self.bidding_area = bidding_area # a 'BiddingArea' object 308 self.bidding_area = bidding_area # a 'BiddingArea' object
309 self.value = value 309 self.value = value
310 310
311 def __str__(self): 311 def __str__(self):
312 return ( 312 return (
313 f"{self.record_created_time} {self.start_time} " 313 f"{self.record_created_time} {self.start_time} "
314 f"{self.production_type} {self.bidding_area} {self.value}" 314 f"{self.production_type} {self.bidding_area} {self.value}"
315 ) 315 )
316 316
317 Base.metadata.create_all(engine) # create tables 317 Base.metadata.create_all(engine) # create tables
318 </code> 318 </code>
@@ -337,27 +337,27 @@
337 session.add(bidding_area1) 337 session.add(bidding_area1)
338 338
339 session.add_all( 339 session.add_all(
340 [ 340 [
341 BiddingArea("NO2", "Elspot NO2"), 341 BiddingArea("NO2", "Elspot NO2"),
342 BiddingArea("NO3", "Elspot NO3"), 342 BiddingArea("NO3", "Elspot NO3"),
343 BiddingArea("NO4", "Elspot NO4"), 343 BiddingArea("NO4", "Elspot NO4"),
344 BiddingArea("NO5", "Elspot NO5"), 344 BiddingArea("NO5", "Elspot NO5"),
345 ] 345 ]
346 ) 346 )
347 347
348 production_type_B37 = ProductionType("B37", "Thermal unspecified") 348 production_type_B37 = ProductionType("B37", "Thermal unspecified")
349 production_type_B30 = ProductionType("B30", "Wind unspecified") 349 production_type_B30 = ProductionType("B30", "Wind unspecified")
350 350
351 session.add_all( 351 session.add_all(
352 [ 352 [
353 ProductionType("B19", "Wind Onshore"), 353 ProductionType("B19", "Wind Onshore"),
354 ProductionType("B10", "Hydro-electric pure pumped storage head installation"), 354 ProductionType("B10", "Hydro-electric pure pumped storage head installation"),
355 ProductionType("B11", "Hydro Run-of-river head installation"), 355 ProductionType("B11", "Hydro Run-of-river head installation"),
356 ProductionType("B12", "Hydro-electric storage head installation"), 356 ProductionType("B12", "Hydro-electric storage head installation"),
357 ProductionType("A04", "Generation"), 357 ProductionType("A04", "Generation"),
358 production_type_B37, 358 production_type_B37,
359 production_type_B30, 359 production_type_B30,
360 ] 360 ]
361 ) 361 )
362 </code> 362 </code>
363 </pre> 363 </pre>
@@ -371,21 +371,21 @@
371 # adding some production plans... 371 # adding some production plans...
372 372
373 session.add( 373 session.add(
374 ProductionPlan( 374 ProductionPlan(
375 datetime.datetime.now(), 375 datetime.datetime.now(),
376 datetime.datetime(2022, 11, 2, 1, 0), 376 datetime.datetime(2022, 11, 2, 1, 0),
377 production_type_B37, 377 production_type_B37,
378 bidding_area1, 378 bidding_area1,
379 decimal.Decimal("80.5"), 379 decimal.Decimal("80.5"),
380 ) 380 )
381 ) 381 )
382 382
383 production_plan2 = ProductionPlan( 383 production_plan2 = ProductionPlan(
384 datetime.datetime.now(), 384 datetime.datetime.now(),
385 datetime.datetime(2022, 11, 2, 2, 0), 385 datetime.datetime(2022, 11, 2, 2, 0),
386 production_type_B37, 386 production_type_B37,
387 bidding_area1, 387 bidding_area1,
388 decimal.Decimal("90.5"), 388 decimal.Decimal("90.5"),
389 ) 389 )
390 390
391 session.add(production_plan2) 391 session.add(production_plan2)
@@ -427,17 +427,17 @@
427 427
428 # return production plans with production type 'B37' 428 # return production plans with production type 'B37'
429 session.query(ProductionPlan).filter( 429 session.query(ProductionPlan).filter(
430 ProductionPlan.production_type_id == ProductionType.production_type_id 430 ProductionPlan.production_type_id == ProductionType.production_type_id
431 ).filter(ProductionType.code == "B37").all() 431 ).filter(ProductionType.code == "B37").all()
432 session.query(ProductionPlan).join(ProductionType).filter(ProductionType.code == "B37").all() 432 session.query(ProductionPlan).join(ProductionType).filter(ProductionType.code == "B37").all()
433 session.query(ProductionPlan).filter(ProductionPlan.production_type == production_type_B37).all() 433 session.query(ProductionPlan).filter(ProductionPlan.production_type == production_type_B37).all()
434 session.query( 434 session.query(
435 ProductionPlan 435 ProductionPlan
436 ).from_statement( 436 ).from_statement(
437 text( 437 text(
438 "SELECT pp.* FROM production_plans pp, production_types pt " 438 "SELECT pp.* FROM production_plans pp, production_types pt "
439 "WHERE pp.production_type_id = pt.production_type_id AND pt.code=:code" 439 "WHERE pp.production_type_id = pt.production_type_id AND pt.code=:code"
440 ) 440 )
441 ).params(code="B37").all() 441 ).params(code="B37").all()
442 </code> 442 </code>
443 </pre> 443 </pre>
@@ -449,29 +449,28 @@
449 449
450 </div> 450 </div>
451 </div> 451 </div>
452
453 <script src="dist/reveal.js"></script>
454 <script src="plugin/zoom/zoom.js"></script>
455 <script src="plugin/notes/notes.js"></script>
456 <script src="plugin/search/search.js"></script>
457 <script src="plugin/markdown/markdown.js"></script>
458 <script src="plugin/highlight/highlight.js"></script>
459 <script>
460 452
461 // Also available as an ES module, see: 453 <script src="dist/reveal.js"></script>
462 // https://revealjs.netlify.app/initialization/ 454 <script src="plugin/zoom/zoom.js"></script>
463 Reveal.initialize({ 455 <script src="plugin/notes/notes.js"></script>
464 controls: true, 456 <script src="plugin/search/search.js"></script>
465 progress: true, 457 <script src="plugin/markdown/markdown.js"></script>
466 center: true, 458 <script src="plugin/highlight/highlight.js"></script>
467 hash: true, 459 <script>
460
461 // Also available as an ES module, see:
462 // https://revealjs.com/initialization/
463 Reveal.initialize({
464 controls: true,
465 progress: true,
466 center: true,
467 hash: true,
468 468
469 // Learn about plugins: https://revealjs.netlify.app/plugins/ 469 // Learn about plugins: https://revealjs.com/plugins/
470 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ] 470 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
471 }); 471 });
472 Reveal.configure({ pdfSeparateFragments: false });
473 472
474 </script> 473 </script>
475 474
476 </body> 475 </body>
477</html> 476</html>
diff --git a/reveal.js/timetravel.html b/reveal.js/timetravel.html
index e073f92..b297e2f 100644
--- a/reveal.js/timetravel.html
+++ b/reveal.js/timetravel.html
@@ -1,127 +1,127 @@
1<!doctype html> 1<!doctype html>
2<html lang="en"> 2<html lang="en">
3 <head> 3 <head>
4 <meta charset="utf-8"> 4 <meta charset="utf-8">
5 <title>Solving problems using time travel</title> 5 <title>Solving problems using time travel</title>
6 <meta name="author" content="Simeon Simeonov"> 6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes"> 7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"> 8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0"> 9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 10
11 <link rel="stylesheet" href="dist/reset.css"> 11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css"> 12 <link rel="stylesheet" href="dist/reveal.css">
13 13
14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme"> 14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme">
15 15
16 <!-- Theme used for syntax highlighting of code --> 16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme"> 17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> --> 18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head> 19 </head>
20 <body> 20 <body>
21 <div class="reveal"> 21 <div class="reveal">
22 22
23 <!-- Any section element inside of this container is displayed as a slide --> 23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides"> 24 <div class="slides">
25 25
26 <section> 26 <section>
27 <h2>Solving problems using <em>time travel</em></h2> 27 <h2>Solving problems using <em>time travel</em></h2>
28 <h4>Beryl @ Fifty</h4> 28 <h4>Beryl @ Fifty</h4>
29 </br> 29 </br>
30 <p><small>Tomas Robertson (team ACE-OL) &amp; Simeon Simeonov (team Forecasts)</small></p> 30 <p><small>Tomas Robertson (team ACE-OL) &amp; Simeon Simeonov (team Forecasts)</small></p>
31 </section> 31 </section>
32 32
33 33
34 <section> 34 <section>
35 <h2>What is <em>time travel</em>???</h2> 35 <h2>What is <em>time travel</em>???</h2>
36 <br/> 36 <br/>
37 <p><em>Time travel</em> - our ability to look at our input data at the state it was at a specific point in time (not only at its last state).</p> 37 <p><em>Time travel</em> - our ability to look at our input data at the state it was at a specific point in time (not only at its last state).</p>
38 <p><em>Time travel</em> is achieved by adding the <em>record_created_time</em> column to our DB tables and storing Kafka's <em>created_time</em> value (converted to UTC).</p> 38 <p><em>Time travel</em> is achieved by adding the <em>record_created_time</em> column to our DB tables and storing Kafka's <em>created_time</em> value (converted to UTC).</p>
39 <br/> 39 <br/>
40 <p>The main concepts around how and why were presented by Peter Sandberg.</p> 40 <p>The main concepts around how and why were presented by Peter Sandberg.</p>
41 </section> 41 </section>
42 42
43 <section> 43 <section>
44 <h2>Solving problems</h2> 44 <h2>Solving problems</h2>
45 <br/> 45 <br/>
46 <p>The following spike was observed 2021-10-06 around 10:15 CET @ NO4</p> 46 <p>The following spike was observed 2021-10-06 around 10:15 CET @ NO4</p>
47 <img src="images/timetravel/ace_ol_spike.png"></img> 47 <img src="images/timetravel/ace_ol_spike.png"></img>
48 </section> 48 </section>
49 49
50 <section> 50 <section>
51 <h2>Solving problems (cont)</h2> 51 <h2>Solving problems (cont)</h2>
52 <br/> 52 <br/>
53 <p>No spikes shown in Grafana</p> 53 <p>No spikes shown in Grafana</p>
54 <img src="images/timetravel/ace_ol_spike_grafana.png"></img> 54 <img src="images/timetravel/ace_ol_spike_grafana.png"></img>
55 </section> 55 </section>
56 56
57 <section> 57 <section>
58 <h2>Solving problems with <em>time travel</em></h2> 58 <h2>Solving problems with <em>time travel</em></h2>
59 <br/> 59 <br/>
60 <pre data-id="code-animation"> 60 <pre data-id="code-animation">
61 <code class="sql" data-trim type="text/template"> 61 <code class="sql" data-trim type="text/template">
62 SELECT record_created_time, start_time, value 62 SELECT record_created_time, start_time, value
63 FROM misc.app_odin_ace_ol_ba_10s_avro_v01 63 FROM misc.app_odin_ace_ol_ba_10s_avro_v01
64 WHERE bidding_area_name = 'NO4' AND start_time = '2021-10-06 08:14:10' 64 WHERE bidding_area_name = 'NO4' AND start_time = '2021-10-06 08:14:10'
65 ORDER BY record_created_time; 65 ORDER BY record_created_time;
66 </code> 66 </code>
67 </pre> 67 </pre>
68 68
69 <table> 69 <table>
70 <thead> 70 <thead>
71 <tr> 71 <tr>
72 <th>record_created_time</th> 72 <th>record_created_time</th>
73 <th>start_time</th> 73 <th>start_time</th>
74 <th>value</th> 74 <th>value</th>
75 </tr> 75 </tr>
76 </thead> 76 </thead>
77 <tbody> 77 <tbody>
78 <tr> 78 <tr>
79 <td>2021-10-06 08:15:52.258</td> 79 <td>2021-10-06 08:15:52.258</td>
80 <td>2021-10-06 08:14:10</td> 80 <td>2021-10-06 08:14:10</td>
81 <td>-370.37683609008127</td> 81 <td>-370.37683609008127</td>
82 </tr> 82 </tr>
83 <tr> 83 <tr>
84 <td>2021-10-06 08:19:06.222</td> 84 <td>2021-10-06 08:19:06.222</td>
85 <td>2021-10-06 08:14:10</td> 85 <td>2021-10-06 08:14:10</td>
86 <td>25.183745117193457</td> 86 <td>25.183745117193457</td>
87 </tr> 87 </tr>
88 <tr> 88 <tr>
89 <td>2021-10-06 08:21:56.207</td> 89 <td>2021-10-06 08:21:56.207</td>
90 <td>2021-10-06 08:14:10</td> 90 <td>2021-10-06 08:14:10</td>
91 <td>34.78731922151518</td> 91 <td>34.78731922151518</td>
92 </tr> 92 </tr>
93 </tbody> 93 </tbody>
94 </table> 94 </table>
95 </section> 95 </section>
96 96
97 <section> 97 <section>
98 <h1>Q &amp; A</h1> 98 <h1>Q &amp; A</h1>
99 </section> 99 </section>
100 100
101 </div> 101 </div>
102 </div> 102 </div>
103
104 <script src="dist/reveal.js"></script>
105 <script src="plugin/zoom/zoom.js"></script>
106 <script src="plugin/notes/notes.js"></script>
107 <script src="plugin/search/search.js"></script>
108 <script src="plugin/markdown/markdown.js"></script>
109 <script src="plugin/highlight/highlight.js"></script>
110 <script>
111 103
112 // Also available as an ES module, see: 104 <script src="dist/reveal.js"></script>
113 // https://revealjs.netlify.app/initialization/ 105 <script src="plugin/zoom/zoom.js"></script>
114 Reveal.initialize({ 106 <script src="plugin/notes/notes.js"></script>
115 controls: true, 107 <script src="plugin/search/search.js"></script>
116 progress: true, 108 <script src="plugin/markdown/markdown.js"></script>
117 center: true, 109 <script src="plugin/highlight/highlight.js"></script>
118 hash: true, 110 <script>
119 111
120 // Learn about plugins: https://revealjs.netlify.app/plugins/ 112 // Also available as an ES module, see:
121 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ] 113 // https://revealjs.com/initialization/
122 }); 114 Reveal.initialize({
123 115 controls: true,
124 </script> 116 progress: true,
117 center: true,
118 hash: true,
125 119
126 </body> 120 // Learn about plugins: https://revealjs.com/plugins/
121 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
122 });
123
124 </script>
125
126 </body>
127</html> 127</html>
generated by cgit v1.3 (git 2.53.0) at 2026-09-25 09:00:56 +0000