From be44243136d710eec0345f8459a64da377bab357 Mon Sep 17 00:00:00 2001 From: Simeon Simeonov Date: Sun, 29 Mar 2026 13:43:38 +0200 Subject: Re-format reveal.js slides and add templates --- notebooks/python/cryptographic_primitives.ipynb | 261 ++++ notebooks/python/python_3_8_to_3_11.ipynb | 2 +- notebooks/python/python_intro.ipynb | 120 +- notebooks/python/python_oo.ipynb | 151 +- reveal.js/cryptographic_primitives.html | 494 +++++++ reveal.js/demo.html | 443 ++++-- reveal.js/dist/theme/statnett_compact.css | 307 ++++ reveal.js/free_software.html | 70 +- reveal.js/git.html | 73 +- reveal.js/images/cryptographic_primitives/GCM.svg | 1500 ++++++++++++++++++++ reveal.js/images/cryptographic_primitives/hmac.svg | 165 +++ .../modes_of_operation.png | Bin 0 -> 195700 bytes .../dataporten/dataporten_the_big_picture1.png | Bin 9698 -> 0 bytes reveal.js/images/dataporten/flow1.png | Bin 22182 -> 0 bytes reveal.js/images/gateway/dashboard01.png | Bin 139944 -> 0 bytes reveal.js/images/gateway/portal01.png | Bin 97986 -> 0 bytes reveal.js/images/gateway/portal02.png | Bin 91353 -> 0 bytes reveal.js/images/mqprod/flow01.png | Bin 40772 -> 0 bytes reveal.js/mqprod.html | 248 ---- reveal.js/postgresql_tuning.html | 258 ++-- reveal.js/rabbitmq.html | 311 ---- reveal.js/sqlalchemy.html | 927 ++++++------ reveal.js/template.html | 55 + reveal.js/template_statnett.html | 56 + reveal.js/timetravel.html | 127 -- 25 files changed, 4029 insertions(+), 1539 deletions(-) create mode 100644 notebooks/python/cryptographic_primitives.ipynb create mode 100644 reveal.js/cryptographic_primitives.html create mode 100644 reveal.js/dist/theme/statnett_compact.css mode change 100755 => 100644 reveal.js/git.html create mode 100644 reveal.js/images/cryptographic_primitives/GCM.svg create mode 100644 reveal.js/images/cryptographic_primitives/hmac.svg create mode 100644 reveal.js/images/cryptographic_primitives/modes_of_operation.png delete mode 100644 reveal.js/images/dataporten/dataporten_the_big_picture1.png delete mode 100644 reveal.js/images/dataporten/flow1.png delete mode 100644 reveal.js/images/gateway/dashboard01.png delete mode 100644 reveal.js/images/gateway/portal01.png delete mode 100644 reveal.js/images/gateway/portal02.png delete mode 100644 reveal.js/images/mqprod/flow01.png delete mode 100644 reveal.js/mqprod.html delete mode 100644 reveal.js/rabbitmq.html create mode 100644 reveal.js/template.html create mode 100644 reveal.js/template_statnett.html delete mode 100644 reveal.js/timetravel.html diff --git a/notebooks/python/cryptographic_primitives.ipynb b/notebooks/python/cryptographic_primitives.ipynb new file mode 100644 index 0000000..4d96549 --- /dev/null +++ b/notebooks/python/cryptographic_primitives.ipynb @@ -0,0 +1,261 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 12, + "id": "fedd7d1a-36a9-4255-8305-a720dfb7b500", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b\"{\\x8ea\\xf8wu!\\xeb\\xc4\\xc9\\xcb|\\\\\\xf1K\\xd5t\\x80i\\xbcD\\xb6!\\xfc\\xee\\x06lQ'V0\\xc6\"\n", + "7b8e61f8777521ebc4c9cb7c5cf14bd5748069bc44b621fcee066c51275630c6\n", + "b'e45h+Hd1IevEyct8XPFL1XSAabxEtiH87gZsUSdWMMY='\n" + ] + } + ], + "source": [ + "# hashes\n", + "\n", + "import base64\n", + "import hashlib\n", + "\n", + "hashed_result = hashlib.sha256(b\"This is a tesu\")\n", + "print(hashed_result.digest())\n", + "print(hashed_result.hexdigest())\n", + "print(base64.b64encode(hashed_result.digest()))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9082c388-1c30-46dc-8897-15bf5db255c8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "password_hash1 = 'pbkdf2_sha256$100000$j2jYEb1TvX/wh5JEAYGag6h+hRNTGe3rjyv5pegVBu4=$7cV7MljnJJYuGndge3jozwFP28aP8Gfj+QFGvv5bpfo='\n", + "password_hash2 = 'pbkdf2_sha256$100000$8+PsThJmJXNAbQ/zaR5AgRA/CuNv8L1uPUXRNuWHc9k=$Kf07CSCVQOTQrvWX5l/ZBdA7zsaEFH247QjUaewpxaU='\n" + ] + } + ], + "source": [ + "# PBKDF2\n", + "\n", + "import base64\n", + "import hashlib\n", + "import os\n", + "\n", + "\n", + "def get_password_hash(plaintext_password: str) -> str:\n", + " \"\"\"\n", + " Returns a complete password hash based on `plaintext_password`\n", + "\n", + " :return: The hashed version of `plaintext_password`\n", + " :rtype: str\n", + " \"\"\"\n", + " # parameters typycally sent as arguments or fetched from config\n", + " hash_algo = \"sha256\" # the hash algorithm to use for HMAC\n", + " iterations = 100000 # amount of iterations\n", + "\n", + " salt = os.urandom(32) # 32 bytes of random data\n", + "\n", + " key = hashlib.pbkdf2_hmac(hash_algo, plaintext_password.encode(), salt, iterations)\n", + "\n", + " return (\n", + " f\"pbkdf2_{hash_algo}${iterations}$\"\n", + " f\"{base64.b64encode(salt).decode(\"ascii\")}$\"\n", + " f\"{base64.b64encode(key).decode(\"ascii\")}\"\n", + " )\n", + "\n", + "\n", + "def password_matches(plaintext_password: str, password_hash: str) -> bool:\n", + " \"\"\"Returns True if `plaintext_password` matches `password_hash`, False otherwise\"\"\"\n", + " hash_tokens = password_hash.split(\"$\")\n", + " hash_algo = hash_tokens[0].split(\"_\")[1]\n", + " iterations = int(hash_tokens[1])\n", + " salt = base64.b64decode(hash_tokens[2])\n", + " key = hashlib.pbkdf2_hmac(hash_algo, plaintext_password.encode(), salt, iterations)\n", + " return key == base64.b64decode(hash_tokens[3])\n", + "\n", + "\n", + "password_hash1 = get_password_hash(\"didn't expect a Spanish Inquisition!\")\n", + "password_hash2 = get_password_hash(\"didn't expect a Spanish Inquisition!\")\n", + "print(f\"{password_hash1 = }\")\n", + "print(f\"{password_hash2 = }\")\n", + "\n", + "# print(password_matches(\"didn't expect a Spanish Inquisition!\", 'pbkdf2_sha256$100000$JfwrS72mFimypbwhJ/NCyds2dkXtknvfWM5AM3+Z5GQ=$k8YA8csSJZhuYW0um7utq3+lSX5pTAQPd9dukObWrPo='))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1274325e-f670-4af8-a0bc-2ff7f668f769", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('gxk9W0XFgbzbjRWB7zCPjbWs3', '00000bf5db2c3c6484904bf5e8314aeb123f5594af1f283fb62e14c7d196e84c')\n" + ] + } + ], + "source": [ + "# Proof of work\n", + "\n", + "import hashlib\n", + "import random\n", + "import string\n", + "\n", + "VALID_CHARS = string.ascii_letters + string.digits # valid characters for response\n", + "\n", + "\n", + "def pow_solve_rnd(challenge: str, starting_chars: str = \"00000\", answer_size: int = 25) -> tuple:\n", + " \"\"\"\n", + " Solves a POW challenge for SHA-256 using `random.choice`\n", + "\n", + " :return: (response, SHA-256 digest) tuple that solves the challenge\n", + " :rtype: tuple\n", + " \"\"\"\n", + " while True:\n", + " attempt = ''.join(\n", + " [random.choice(VALID_CHARS) for x in range(answer_size)])\n", + " dig = hashlib.sha256(('{0}:{1}'.format(challenge, attempt)).encode()).hexdigest()\n", + " if dig.startswith(starting_chars):\n", + " return attempt, dig\n", + "\n", + "print(f\"{pow_solve_rnd('2024-08-07')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "6752c055-7e49-46c4-858b-599d069a7e77", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "enc-val$2$6ppVDmiJi0P9BnHuzwFUPA+vV/Gsb4INe2O9E/Ma2uI=$6879D8rw8645nbQ/NQtY3Q+AwKTWpd1P6Q==\n", + "Invalid tag when decrypting enc-val$2$tmuH3MTDQ9vMbxJ1pZUb0rQlF43Lgr/AYfkA+vSOZs0=$5CVt3VrxhuuOOVt1vCXeGpbhNh/20IcFNg==\n", + "None\n" + ] + } + ], + "source": [ + "# Encryption / decryption using AES-256-GCM\n", + "\n", + "import hashlib\n", + "import os\n", + "\n", + "from cryptography.exceptions import InvalidTag\n", + "from cryptography.hazmat.primitives.ciphers.aead import AESGCM\n", + "\n", + "\n", + "def encrypt(password: str, data: str) -> str:\n", + " \"\"\"\n", + " Encrypts `data` using `password` and AES-256-GCM.\n", + "\n", + " The output string is in the following format:\n", + " enc-val$`version-num`$`bas64-salt`$`base64-encrypted_data`\n", + "\n", + " :param password: The password to generate the key with\n", + " :type password: str\n", + "\n", + " :param data: The data to be encrypted\n", + " :type data: str\n", + "\n", + " :return: The output string\n", + " :rtype: str\n", + " \"\"\"\n", + " data_bytes = data.encode()\n", + " salt = os.urandom(32)\n", + " aesgcm = AESGCM(\n", + " hashlib.scrypt(\n", + " password.encode(), salt=salt, n=2**14, r=8, p=1, dklen=32\n", + " )\n", + " )\n", + " nonce = salt[:12]\n", + " # padding_length_bytes = b\"-1\" # no padding used 2 bytes \"sign\"\n", + " edata = aesgcm.encrypt(\n", + " nonce, data_bytes, salt\n", + " )\n", + " return (\n", + " f\"enc-val$2${base64.b64encode(salt).decode()}${base64.b64encode(edata).decode()}\"\n", + " )\n", + "\n", + "\n", + "def decrypt(password: str, edata: str) -> str:\n", + " \"\"\"\n", + " Decrypts `edata` using `password`.\n", + "\n", + " `edata` is in the following format:\n", + " enc-val$`version-num`$`bas64-salt`$`base64-encrypted_data`\n", + "\n", + " :param password: The password to generate the key with\n", + " :type password: str\n", + "\n", + " :param edata: The data to be decrypted\n", + " :type edata: str\n", + "\n", + " :return: The output string / decrypted data\n", + " :rtype: str\n", + " \"\"\"\n", + " # check for supported versions first...\n", + "\n", + " try:\n", + " salt, data = (base64.b64decode(t) for t in edata[10:].split('$'))\n", + " nonce = salt[:12]\n", + " aesgcm = AESGCM(\n", + " hashlib.scrypt(\n", + " password.encode(), salt=salt, n=2**14, r=8, p=1, dklen=32\n", + " )\n", + " )\n", + "\n", + " # decrypt\n", + " data = aesgcm.decrypt(nonce, data, salt)\n", + " return data.decode()\n", + " except InvalidTag:\n", + " print(f\"Invalid tag when decrypting {edata}\")\n", + "\n", + "print(f\"{encrypt('my passphrase', 'my secret')}\")\n", + "print(f\"{decrypt('my passphrase', 'enc-val$2$tmuH3MTDQ9vMbxJ1pZUb0rQlF43Lgr/AYfkA+vSOZs0=$5CVt3VrxhuuOOVt1vCXeGpbhNh/20IcFNg==')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ccfb453-d3fd-4383-a01d-f81b19f2cb82", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/python_3_8_to_3_11.ipynb b/notebooks/python/python_3_8_to_3_11.ipynb index b8724e8..016c108 100644 --- a/notebooks/python/python_3_8_to_3_11.ipynb +++ b/notebooks/python/python_3_8_to_3_11.ipynb @@ -834,7 +834,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/python/python_intro.ipynb b/notebooks/python/python_intro.ipynb index f129ffe..f101400 100644 --- a/notebooks/python/python_intro.ipynb +++ b/notebooks/python/python_intro.ipynb @@ -48,6 +48,19 @@ "- Calling C from Python: Cython, CFFI, ctypes" ] }, + { + "cell_type": "markdown", + "id": "779bac39-4646-4b80-9970-343a3daef1e7", + "metadata": {}, + "source": [ + "## Important PEPs\n", + "\n", + "- [PEP0](https://peps.python.org/) - Index of Python Enhancement Proposals\n", + "- [PEP8](https://peps.python.org/pep-0008/) - Style Guide for Python Code\n", + "- [PEP257](https://peps.python.org/pep-0257/) - Docstring Conventions\n", + "- [PEP484](https://peps.python.org/pep-0484/) - Type Hints" + ] + }, { "cell_type": "markdown", "id": "6d1edcfc-b001-4393-9c24-72adc9b5fccf", @@ -72,7 +85,7 @@ }, { "cell_type": "markdown", - "id": "4f144f73-51c1-419d-b71e-ee513f68f41c", + "id": "81573ed5-b76e-4726-9f44-7dc017fd0896", "metadata": {}, "source": [ "## Built-in functions\n", @@ -81,8 +94,6 @@ "\n", "[https://docs.python.org/3/library/functions.html](https://docs.python.org/3/library/functions.html)\n", "\n", - "- `dir([obj])` - returns a list of valid attributes for that object\n", - "\n", "- `id(obj)` - returns the \"identity\" of an object - an integer which is guaranteed to be unique\n", "\n", "- `print(...)` - prints objects to a text stream\n", @@ -92,6 +103,12 @@ "- `type(obj)` - returns the type of an object" ] }, + { + "cell_type": "markdown", + "id": "bc180243-fe11-405c-beb9-83db4f793a05", + "metadata": {}, + "source": [] + }, { "cell_type": "markdown", "id": "a0c57535-e285-47af-80f9-1d4a16de5f25", @@ -116,26 +133,28 @@ "source": [ "# Common built-in types:\n", "\n", - "s = 'foo' # this is a string / str, same as str('foo'), may be encoded, immutable (s[0] = 'r' is NOT possible)\n", + "my_string = 'foo' # a string / str, same as str('foo'), may be encoded\n", + " # immutable (s[0] = 'r' is NOT possible)\n", "\n", - "b = b'foo' # bytes, same as bytes('foo', 'utf-8'), may be decoded, immutable\n", + "my_bytes = b'foo' # bytes, same as bytes('foo', 'utf-8'), may be decoded, immutable\n", "\n", - "i = 6 # int, same as int('6'), immutable\n", + "my_int = 6 # int, same as int('6'), immutable\n", "\n", - "f = 0.1 # float, same as float('0.1'), immutable, Note!!: Floats have a fixed size,\n", - "# hence they don't necessarily behave they way we expect from math class.\n", + "my_float = 0.1 # float, same as float('0.1'), immutable, Note!!: Floats have a fixed size,\n", + " # hence they don't necessarily behave they way we expect from math class.\n", + "# 24732847234232342892343428.3 == 24732847234232342892343428.1 # True\n", "\n", - "b = False # bool, same as bool(0), bool(''), bool(None)... immutable / constant\n", + "my_bool = False # bool, same as bool(0), bool(''), bool(None)... immutable / constant\n", "\n", - "n = None # NoneType, similar to 'null' in other languages, immutable / constant\n", + "my_none = None # NoneType, similar to 'null' in other languages, immutable / constant\n", "\n", - "l = [1, False, 'foo'] # list, same as list((1, False, 'foo'))\n", + "my_list = [1, False, 'foo'] # list, same as list((1, False, 'foo'))\n", "\n", - "t = (1, False, 'foo') # tuple, same as tuple([1, False, 'foo']), immutable\n", + "my_tuple = (1, False, 'foo') # tuple, same as tuple([1, False, 'foo']), immutable\n", "\n", - "d = {'foo': 1, 'bar': 8} # dict, same as dict(foo=1, bar=8), similar to hash in other languages\n", + "my_dict = {'foo': 1, 'bar': 8} # dict, same as dict(foo=1, bar=8), similar to hash in other languages\n", "\n", - "s = {'foo', 'bar', 1, 1, 4} # set, same as set(['foo', 'bar', 1, 1, 4]), removes duplicates" + "my_set = {'foo', 'bar', 1, 1, 4} # set, same as set(['foo', 'bar', 1, 1, 4]), removes duplicates" ] }, { @@ -203,7 +222,7 @@ "s3[0] # Out: S\n", "s3[-1] # Out: t\n", "s3[1:] # Out: tatnett\n", - "s3[1:-1] # Out: tatnett\n", + "s3[1:-1] # Out: tatnet\n", "s3[-3:] # Out: ett\n", "s3[1:-1:2] # Out: tte\n", "\n", @@ -221,6 +240,29 @@ "b'B\\xc3\\x98!'.decode('utf-8') # utf-8 is implicit in Python 3" ] }, + { + "cell_type": "markdown", + "id": "25d0f9dc-d783-4a23-a3ee-c0b6cee3d4ab", + "metadata": {}, + "source": [ + "## Lists vs. tuples\n", + "\n", + "Tuples are not sumply \"read-only\" lists.\n", + "\n", + "\"Though tuples may seem similar to lists, they are often used in different situations and for different purposes.\n", + "Tuples are immutable, and usually contain a heterogeneous sequence of elements that are accessed via unpacking ... or indexing.\n", + "Lists are mutable, and their elements are usually homogeneous and are accessed by iterating over the list.\n", + "\n", + "```python\n", + "my_tuple = (1,2)\n", + "my_list = [1,2] \n", + "\n", + "# tuples are immutable and hashable, hence they can be used as keys in mapping objects as dicts\n", + "d = {my_tuple: 1} # OK\n", + "d = {my_list: 1} # Error\n", + "```" + ] + }, { "cell_type": "markdown", "id": "6b2bc8fa-a5dc-4a58-92b2-abfb19bfe2ed", @@ -302,7 +344,7 @@ "id": "40db50e4-fb7a-4a77-a074-bc4413f27212", "metadata": {}, "source": [ - "## Creating and maintaining a Python environment (cont...)\n", + "## Creating and maintaining a Python environment - goals\n", "\n", "Desired qualities for a flexible Python environment:\n", "\n", @@ -324,10 +366,11 @@ "id": "c661bf56-add0-402c-a629-a857f81d0758", "metadata": {}, "source": [ - "## Creating and maintaining a Python environment (cont...)\n", + "## Creating and maintaining a Python environment - using devbox / WSL / UNIX systems directly\n", "\n", "Exploting the operating system can be done by:\n", "\n", + "- using the official Python packages that are maintained by the OS (Ubuntu packages in the case of WSL @ Statnett or a Red Hat enterprise VM @ Statnett) is often good enough\n", "- (re)defining `PYTHONPATH`\n", "- using symlinks to point at packages placed at different locations" ] @@ -337,7 +380,7 @@ "id": "db8952aa-0944-4644-9626-7460786fd72a", "metadata": {}, "source": [ - "## Creating and maintaining a Python environment (cont...)\n", + "## Creating and maintaining a Python environment - using the Python virtual environment - *venv*\n", "\n", "Using venv can be done by directly invoking python:\n", "\n", @@ -365,10 +408,13 @@ "id": "299641e6-3d87-4a86-a124-eb49edb75b4b", "metadata": {}, "source": [ - "## Creating and maintaining a Python environment (cont...)\n", + "## Creating and maintaining a Python environment - using Poetry\n", "\n", "Poetry [https://python-poetry.org](https://python-poetry.org) is the prefered environment and dependency management tool at Statnett.\n", "\n", + "Although Poetry creates and uses vierual environment(s) behind the scenes,\n", + "it is centered around the concept of \"projects\" and not the virtual environment itself\n", + "\n", "```bash\n", "# create project and a virtual environment from scratch\n", "poetry new my-project\n", @@ -411,7 +457,7 @@ { "data": { "text/plain": [ - "-42990858669078524" + "-3158428850515418558" ] }, "execution_count": 15, @@ -435,8 +481,8 @@ "hash(t) # returns f.i. -6333845781340707986\n", "# hash(l) # TypeError: unhashable type: 'list'\n", "\n", - "s = 'the long and winding road'\n", - "s2 = 'the long and winding road'\n", + "s = \"the long and winding road\"\n", + "s2 = \"the long and winding road\"\n", "\n", "# check if s and s2 are the same object:\n", "id(s) # Out: 139858905258704\n", @@ -816,7 +862,7 @@ "# the function will behave differently depending on what parameters were\n", "# used when calling its enclosing function (factory function)\n", "\n", - "def get_multiplier_of(base: int) -> str:\n", + "def get_multiplier_of(base: int):\n", " \"\"\"the function enclosing its nested functions\"\"\"\n", " # this function is the enclosing function of the function `multiplier_function`\n", "\n", @@ -958,19 +1004,7 @@ "execution_count": 22, "id": "4346260d-76af-437e-9c10-1c1d0c478695", "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'is_admin' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[22], line 25\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m decorated\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m api_access_decorator\n\u001b[1;32m 24\u001b[0m \u001b[38;5;129m@requires_access\u001b[39m(access_secret\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mb28cfeaa65b73cf\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;129m@is_admin\u001b[39m\n\u001b[1;32m 26\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msensitive_function\u001b[39m(data, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 27\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"very sensitive function\"\"\"\u001b[39;00m\n\u001b[1;32m 28\u001b[0m db\u001b[38;5;241m.\u001b[39msave(data)\n", - "\u001b[0;31mNameError\u001b[0m: name 'is_admin' is not defined" - ] - } - ], + "outputs": [], "source": [ "# Decorators (cont ...) - a complete example\n", "\n", @@ -996,7 +1030,7 @@ "\n", "\n", "@requires_access(access_secret='b28cfeaa65b73cf')\n", - "@is_admin\n", + "# @is_admin - decorators can be \"chained\"\n", "def sensitive_function(data, **kwargs):\n", " \"\"\"very sensitive function\"\"\"\n", " db.save(data)" @@ -1030,11 +1064,19 @@ "\n", "\n", "# modern Python >= 3.6 f-strings\n", - "f'{s} - {i} - {f:5.2f}' # Out: 'another string - 27 - 6.58'\n", + "f\"{s} - {i} - {f:5.2f}\" # Out: 'another string - 27 - 6.58'\n", "``` \n", "\n", "See https://docs.python.org/3/library/string.html#formatspec for the complete format specification" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23972592-e838-4fa7-81f1-ca3ca129e757", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1053,7 +1095,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/notebooks/python/python_oo.ipynb b/notebooks/python/python_oo.ipynb index e38d7f3..1ef2a72 100644 --- a/notebooks/python/python_oo.ipynb +++ b/notebooks/python/python_oo.ipynb @@ -66,7 +66,55 @@ "\n", "- improving modularity\n", "\n", - "- providing foundation for a more intuitive design" + "- providing foundation for a more intuitive design\n", + "\n", + "A pseudo code example:\n", + "\n", + "```c\n", + "/* crating and drawing a (pseudo) widget in C */\n", + "my_widget = mylibrary_widget_init(); /* a struct of the type widget */\n", + "mylibrary_widget_draw(my_widget);\n", + "```\n", + "\n", + "```python\n", + "# creating and drawing a (pseudo) widget in Python\n", + "import mylibrary\n", + "\n", + "my_widget = mylibrary.Widget()\n", + "my_widget.draw()\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "878cb805-a2d7-46a3-a34a-0507495420f7", + "metadata": {}, + "source": [ + "# Do I need to know / understand OOP when I am programming in Python?\n", + "\n", + "Most definitely - **yes**.\n", + "\n", + "While using programming language that is not object-oriented (like *C* and / or *Rust*) is always possible,\n", + "when using Python there are no *sane* alternatives that have ever been demonstrated:\n", + "\n", + "- Understanding OO behaviour and syntax is a must in order to understand Python code in general\n", + "- OOP is at the core of Python's design - everything in Python is an object\n", + "- Good programming and design is not only about solving a particular task. It is about reusing code and making your code reusable in turn\n", + "- Following the priciples described above is more challenging when not using OO in general." + ] + }, + { + "cell_type": "markdown", + "id": "b83dc815-ed51-4e5a-a30e-fa0252217194", + "metadata": {}, + "source": [ + "# I am doing only simple things with Python. Do I still need to learn OOP?\n", + "\n", + "Most definitely - **yes**.\n", + "\n", + "- A simple task provides a great opportunity to learn and practice OOP\n", + "- Using OOP **does not mean overcomplicating** your code. When applied correctly - it means the opposite\n", + "- Doing \"simple things\" exclusively is not very ambitious approach to programming and learning in general" ] }, { @@ -109,13 +157,20 @@ "\n", "- object - an instance of a class that may contain its own attributes as well as references to its class' attributes\n", "\n", - "- attribute - variable, property, function defined in the class and present in its instances\n", + "- attribute - variable, property, function defined in the class and present in its instances - in Python: any name following a dot. Attributes may be *read-only* or *writable*. In the latter case, assignment to attributes is possible.\n", "\n", "- class variable - attribute of which a single copy exists, regardless of how many instances of the class exist\n", "\n", "- object / instance variable, object / instance attribute - attribute for which each instantiated object of the class has a separate copy, or instance\n", "\n", - "- method - member function - function that is an attribute" + "- method - member function - function that is an attribute\n", + "\n", + "```python\n", + "# my_str_obj is an object (instance) of the class str\n", + "my_str_object = str(\"Test\") # or my_str_object = \"Test\"\n", + "my_str_object.lower # attribute that is a function -> method\n", + "my_str_object.lower() # calling the method \"lower\"\n", + "```" ] }, { @@ -218,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 41, "id": "a9c30a5d-aa86-45ab-b897-f051df80f1d8", "metadata": {}, "outputs": [ @@ -226,17 +281,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "car1.get_obj_info_str() = \"I am <__main__.Car object at 0x7f70e643d590> with id 140122876269968 from with id 93964809433504\"\n", - "car2.get_obj_info_str() = \"I am <__main__.Car object at 0x7f70e63ddf10> with id 140122875879184 from with id 93964809433504\"\n", - "car1.model = 'BMW', car1.reg_nr = 'EC76183', car1.extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140122875882432, id(car1.get_obj_info_str) = 140122875878080\n", - "car2.model = 'Scoda', car2.reg_nr = 'BD77655', car2.extras = ['GPSnav'], id(car2.cls_extras) = 140122875882432, id(car2.get_obj_info_str) = 140122875880192\n", - "id(Car.cls_extras) = 140122875882432, id(Car.get_obj_info_str) = 140122875964416\n", - "car1.cls_extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140122875882432\n", - "car2.cls_extras = ['GPSnav', 'Sound system'], id(car2.cls_extras) = 140122875882432\n", + "car1.get_obj_info_str() = \"I am <__main__.Car object at 0x7fd00029ff80> with id 140531332677504 from with id 93887620597168\"\n", + "car2.get_obj_info_str() = \"I am <__main__.Car object at 0x7fd00029ea50> with id 140531332672080 from with id 93887620597168\"\n", + "car1.model = 'BMW', car1.reg_nr = 'EC76183', car1.extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140531337557440, id(car1.get_obj_info_str) = 140531251735488\n", + "car2.model = 'Scoda', car2.reg_nr = 'BD77655', car2.extras = ['GPSnav'], id(car2.cls_extras) = 140531337557440, id(car2.get_obj_info_str) = 140531251728064\n", + "id(Car.cls_extras) = 140531337557440, id(Car.get_obj_info_str) = 140531252010400\n", + "car1.cls_extras = ['GPSnav', 'Sound system'], id(car1.cls_extras) = 140531337557440\n", + "car2.cls_extras = ['GPSnav', 'Sound system'], id(car2.cls_extras) = 140531337557440\n", "True\n", "hasattr(car1, 'import_tax_paid') = True\n", "hasattr(car2, 'import_tax_paid') = False\n", - "id(car1.__class__) = 93964809433504, id(car2.__class__) = 93964809433504, id(Car) = 93964809433504\n" + "id(car1.__class__) = 93887620597168, id(car2.__class__) = 93887620597168, id(Car) = 93887620597168\n" ] } ], @@ -288,7 +343,7 @@ "# - checks if 'attr' is an instance attribute\n", "# - checks if 'attr' is a class attribute (through the method resolution order - MRO)\n", "# - raises AttributeError\n", - "# setter:\n", + "# setter (making the attribute writable):\n", "# - (re)defines an instance attribute\n", "\n", "# Details not covered in this course:\n", @@ -323,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 42, "id": "391e816e-c53b-454b-9e9f-3587234f1fea", "metadata": {}, "outputs": [ @@ -413,7 +468,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 43, "id": "e82feefb-025d-4cbb-a8ac-ddc3cc01269c", "metadata": {}, "outputs": [ @@ -495,7 +550,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 44, "id": "71a90ec6-3cc0-441e-a866-c2c2b9984559", "metadata": {}, "outputs": [ @@ -613,7 +668,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 45, "id": "a4d2735d-d167-4f43-ad55-cb3d13ece2dc", "metadata": {}, "outputs": [ @@ -656,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 46, "id": "a6ead5e6-e987-4bda-bf48-30f91877278d", "metadata": {}, "outputs": [ @@ -715,7 +770,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 47, "id": "11f6a3fb-64d7-40a9-a130-27548ec4b802", "metadata": {}, "outputs": [ @@ -797,7 +852,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 48, "id": "c345a69f-02da-40e7-bb37-c0511b6af096", "metadata": {}, "outputs": [ @@ -806,6 +861,9 @@ "output_type": "stream", "text": [ "9\n", + "9\n", + "140531251904896\n", + "140531251904896\n", "Vector.from_str('1:1:5:6') = Vector(start=Point(x=1, y=1), end=Point(x=5, y=6))\n" ] } @@ -837,7 +895,14 @@ " Point(coordinates[2], coordinates[3]),\n", " )\n", "\n", - "print(Point.get_manhattan_distance(Point(1, 1), Point(5, 6)))\n", + "point1 = Point(1, 1)\n", + "point2 = Point(5, 6)\n", + "print(f\"{point1.get_manhattan_distance(point1, point2)}\") # works, but should not be used in that way\n", + "print(f\"{Point.get_manhattan_distance(point1, point2)}\")\n", + "print(f\"{id(point1.get_manhattan_distance)}\")\n", + "print(f\"{id(point2.get_manhattan_distance)}\")\n", + "# print(f\"{id(point1.from_str)}\") <-- not available as an instance attribute\n", + "\n", "print(f\"{Vector.from_str('1:1:5:6') = }\")" ] }, @@ -890,7 +955,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 49, "id": "44d47e1c-1a06-4577-8db6-6ec78ce5cef8", "metadata": {}, "outputs": [ @@ -903,32 +968,32 @@ "\n", "class TigerShark(Fish)\n", " | TigerShark(weight: int, alive: bool = True, **kwargs)\n", - " | \n", + " |\n", " | Base class for all tiger sharks\n", - " | \n", + " |\n", " | Method resolution order:\n", " | TigerShark\n", " | Fish\n", " | Animal\n", " | builtins.object\n", - " | \n", + " |\n", " | Methods defined here:\n", - " | \n", + " |\n", " | __init__(self, weight: int, alive: bool = True, **kwargs)\n", " | Initialize self. See help(type(self)) for accurate signature.\n", - " | \n", + " |\n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from Animal:\n", - " | \n", + " |\n", " | __dict__\n", - " | dictionary for instance variables (if defined)\n", - " | \n", + " | dictionary for instance variables\n", + " |\n", " | __weakref__\n", - " | list of weak references to the object (if defined)\n", - " | \n", + " | list of weak references to the object\n", + " |\n", " | alive\n", " | getter property alive\n", - " | \n", + " |\n", " | weight\n", " | getter property weight\n", "\n", @@ -1064,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 50, "id": "e6076925-75f0-456c-aed5-7db0a24a67a1", "metadata": {}, "outputs": [ @@ -1169,7 +1234,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 51, "id": "cf79617a-95d2-48f2-be66-fa1fa34e4e04", "metadata": {}, "outputs": [ @@ -1192,14 +1257,14 @@ " | Adam\n", " | Eve\n", " | builtins.object\n", - " | \n", + " |\n", " | Data descriptors inherited from Adam:\n", - " | \n", + " |\n", " | __dict__\n", - " | dictionary for instance variables (if defined)\n", - " | \n", + " | dictionary for instance variables\n", + " |\n", " | __weakref__\n", - " | list of weak references to the object (if defined)\n", + " | list of weak references to the object\n", "\n" ] } @@ -1256,7 +1321,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 52, "id": "95a1d0cc-0006-4ddc-b08d-d0b8b02acfba", "metadata": {}, "outputs": [ @@ -1324,7 +1389,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 53, "id": "38066107-0190-4383-ba14-a4e9cd454a9c", "metadata": {}, "outputs": [ @@ -1418,7 +1483,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/reveal.js/cryptographic_primitives.html b/reveal.js/cryptographic_primitives.html new file mode 100644 index 0000000..4b2e6fc --- /dev/null +++ b/reveal.js/cryptographic_primitives.html @@ -0,0 +1,494 @@ + + + + + Introduction to cryptographic primitives + + + + + + + + + + + + + + + + + +
+ +
+ +
+

Introduction to cryptographic primitives

+
+

Simeon Simeonov - TDE

+
+ + +
+

What are cryptographic primitives?

+
+

Cryptographic primitives are the most basic building blocks in cryptographic systems and protocols.

+

Creating cryptographic routines is very hard, and testing them to be reliable takes a long time, it is essentially never sensible (nor secure) to design a new cryptographic primitive to suit the needs of a new cryptographic system.

+

Since algorithms in this field are not only required to be designed well but also need to be tested well by the cryptologist community, even if a cryptographic routine looks good from a design point of view it might still contain errors. Successfully withstanding such scrutiny gives some confidence (in fact, so far, the only confidence) that the algorithm is indeed secure enough to use. Security proofs for cryptographic primitives are generally not available.

+

When choosing primitive for a cryptographic system, one should always consider if the primitive:

+
    +
  • is open and can be studied by the community
    • +
  • is patented
    • +
+
+ + +
+

Agenda

+
+
+
    +
  • cryptographic hash functions
    •  +
  • cryptographically secure random generators (the very basics)
    •  +
  • symmetric ciphers
    •  +
  • public-key cryptography / asymmetric cryptography
    •  +
+
+
+ + +
+

Cryptographic hash functions

+

A hash function is any function that can be used to map data of arbitrary size to fixed-size values (or a set of fixed-size values)

+

Cryptographic hash functions or cryptographically secure hash functions are hash functions with special properties (making them desirable for cryptographic systems)

+

(Over)simplified list of desired properties:

+
    +
  • the probability of a particular output result (hash value) n for a random input string ("message") is 2^(-n) (as for any good hash), so the hash value can be used as a representative of the message
    • +
  • given a hash value h, it should be difficult to find any message m such that h = hash(m) ("reversing the function")
    • +
  • given an input m1, it should be difficult to find a different input m2 such that hash(m1) = hash(m2) (weak collision resistance)
    • +
  • it should be difficult to find two different messages m1 and m2 such that hash(m1) = hash(m2). Such a pair is called a cryptographic hash collision (strong collision resistance)
    • +
  • if an input is changed slightly (for example, flipping a single bit), the output changes significantly (avalanche effect) - a property also desired in ciphers
    • +
  • being fast is always a nice bonus :)
    • +
+
+ + +
+

MD5

+ + + + + + + + + + + + + + + + + + + + + + + +
DesignersRonald Rivest
Published1992
Digest size128 bits (16 bytes)
Block size512 bits
Broken?Yes (broken collision resistance in 2^18 time)
+

Still in use in lagacy applications and in applications where collision resistance is not needed

+
+ + +
+

SHA-1 (Secure Hash Algorithm 1)

+ + + + + + + + + + + + + + + + + + + + + + + +
DesignersNSA
Published1995
Digest size160 bits (20 bytes)
Block size512 bits
Broken?Yes
+

Still in use. Revision control systems such as Git, Mercurial, and Monotone use SHA-1, not for security, but to identify revisions and to ensure that the data has not changed due to accidental corruption.

+
+ + +
+

SHA-2

+ + + + + + + + + + + + + + + + + + + + + + + +
DesignersNSA
Published2001
Digest sizes224, 256, 384 or 512 bits
Block sizes256 bits (SHA-224 and SHA-256) or 512 bits
Broken?No
+
+ + +
+

SHA-3

+ + + + + + + + + + + + + + + + + + + + + + + +
DesignersGuido Bertoni, Joan Daemen, Michaël Peeters and Gilles van Assche
Published2016
Digest sizearbitrary
Block sizevariable
Broken?No
+

Based on Keccak - winner of the NIST hash function competition after some controversial adjustments. Very fast and flexible.

+
+ + +
+

BLAKE2

+ + + + + + + + + + + + + + + + + + + + + + + +
DesignersJean-Philippe Aumasson, Samuel Neves, Zooko Wilcox-O'Hearn and Christian Winnerlein
Published2008 (BLAKE)
Digest sizeup to 64 bytes (BLAKE2b), up to 32 bytes (BLAKE2s)
Block sizevariable (stream)
Broken?No
+

BLAKE2 is based on Daniel J. Bernstein's ChaCha stream cipher and is extremely fast. BLAKE2b and BLAKE2s are specified in RFC 7693.

+
+ + +
+

Application of cryptographic hash functions

+

Cryptographic hash functions are used for many different things in cryptographic systems and protocols. Few examples:

+
    +
  • can be used directly (f.i. sha256sum - part of GNU coreutils)
    • +
  • message authentication code (MAC)
    • +
  • password hashing / key derivation
    • +
  • proof of work
    • +
+
+ + +
+

HMAC

+

HMAC (hash-based message authentication code) is a specific type of message authentication code (MAC) involving a cryptographic hash function and a secret cryptographic key. It may be used to simultaneously verify both the data integrity and authenticity of a message. An HMAC is a type of keyed hash function that can also be used in a key derivation scheme or a key stretching scheme.

+

HMAC can provide authentication using a shared secret instead of using digital signatures with asymmetric cryptography. It trades off the need for a complex public key infrastructure by delegating the key exchange to the communicating parties, who are responsible for establishing and using a trusted channel to agree on the key prior to communication.

+ +

XOR (eXclusive OR) - is a logical operator

+

With two inputs, XOR is true if and only if the inputs differ (one is true, one is false)

+
+ + +
+

PBKDF2

+

PBKDF2 (Password-Based Key Derivation Function 2) is a key derivation function with a sliding computational cost, used to reduce vulnerability to brute-force attacks.

+

PBKDF2 applies a pseudorandom function, such as HMAC, to the input password or passphrase along with a salt value and repeats the process many times to produce a derived key, which can then be used as a cryptographic key in subsequent operations. The added computational work makes password cracking much more difficult, and is known as key stretching.

+

Demo

+
+ + +
+

Proof of work

+

Demo

+
+ + +
+

Cryptographically secure random generators

+

Most cryptographic applications require random numbers for: generating keys, initialization vectors, nonces (arbitrary numbers that can be used just once), salts, tokens etc, etc.

+

The "quality" of the randomness required for these applications varies. For example, creating a nonce in some protocols needs only uniqueness. On the other hand, the generation of a master key requires a higher quality, such as more entropy.

+

Entropy is obtained from a high-quality source, generally the operating system's randomness API.

+

In practical situations, numbers are needed with more randomness than the available entropy can provide. Also, the processes to extract randomness from a running system are slow in actual practice. In such instances, a cryptographically secure pseudorandom number generator (CSPRNG) can sometimes be used. A CSPRNG can "stretch" the available entropy over more bits.

+
+ + +
+

Cryptographically secure random generators (cont...)

+

CSPRNG must

+
    +
  • pass statistical randomness tests - f.i. given the first k bits of a random sequence, there is no polynomial-time algorithm that can predict the (k+1)th bit with probability of success non-negligibly better than 50%
    • +
  • be attack resistant - f.i. in the event that part or all of its state has been revealed it should be impossible to reconstruct the stream of random numbers prior to the revelation
    • +
+

"Practical" CSPRNG schemes not only include an CSPRNG algorithm, but also a way to initialize ("seed") it while keeping the seed secret.

+
+ + +
+

Symmetric-key algorithms / symmetric ciphers

+

Symmetric ciphers use the same cryptographic keys for both the encryption of plaintext and the decryption of ciphertext.

+

“Anyone, from the most clueless amateur to the best cryptographer, can create an algorithm that he himself can’t break. It’s not even hard. What is hard is creating an algorithm that no one else can break, even after years of analysis. And the only way to prove that is to subject the algorithm to years of analysis by the best cryptographers around.” - Bruce Schneier

+

A "perfect" cipher - the one-time pad has been known since 1882, but is not practically applicable in modern systems:

+
    +
  • generate random stream (pad) with length = len(plaintext)
    • +
  • ciphertext = plaintext XOR pad
    • +
+ +

There two types of modern ciphers:

+
    +
  • block ciphers - operate on fixed-length groups of bits, called blocks (padding may be used to the remaining bits)
    • +
  • stream ciphers - plaintext digits are combined (XORed) with a pseudorandom cipher digit stream (keystream).
    • +
+
+ + +
+

DES (Data Encryption Standard)

+ + + + + + + + + + + + + + + + + + + +
DesignerIBM
Published1975
Key size56 bits
Block size64 bits
+

Not used anymore. 3DES was published in 1981 (with keysize of 112 bits or 168 bits).

+
+ + +
+

Blowfish

+ + + + + + + + + + + + + + + + + + + +
DesignerBruce Schneier
Published1993
Key size32-448 bits
Block size64 bits
+

Schneier has stated that "Blowfish is unpatented, and will remain so in all countries. The algorithm is hereby placed in the public domain, and can be freely used by anyone.". Still in use in legacy applications and notably in the bcrypt password hashing function.

+
+ + +
+

AES (Advanced Encryption Standard - Rijndael)

+ + + + + + + + + + + + + + + + + + + +
DesignerJoan Daemen, Vincent Rijmen
Published1998
Key size128 bits, 192 bits or 256 bits
Block size128 bits
+

Rijndael was thew winner of the NIST AES selection process. Currently the most widely adopted block cipher (both in hardware and software).

+
+ + +
+

Mode of operation for block ciphers

+

A block cipher by itself is only suitable for the secure cryptographic transformation (encryption or decryption) of one fixed-length group of bits called a block. A mode of operation describes how to repeatedly apply a cipher's single-block operation to securely transform amounts of data larger than a block.

+ +
+ + +
+

Galois/counter (GCM)

+

The GCM algorithm provides both data authenticity (integrity) and confidentiality and belongs to the class of authenticated encryption with associated data (AEAD) methods. This means that as input it takes a key K, some plaintext P, and some associated data AD; it then encrypts the plaintext using the key to produce ciphertext C, and computes an authentication tag T from the ciphertext and the associated data (which remains unencrypted). A recipient with knowledge of K, upon reception of AD, C and T, can decrypt the ciphertext to recover the plaintext P and can check the tag T to ensure that neither ciphertext nor associated data were tampered with.

+ +
+ + +
+

Stream ciphers

+

Stream ciphers typically execute at a higher speed than block ciphers and have lower hardware complexity. However, stream ciphers can be susceptible to security breaches, for example, when the same starting state (seed) is used twice.

+

Essentially they behave as pseudorandom functions where they key is / is part of the "seed".

+
+

ChaCha

+ + + + + + + + + + + + + + + +
DesignerDaniel J. Bernstein (djb)
Published2008
Key size128 bits or 256 bits
+
+
ChaCha20-Poly1305
+

ChaCha20-Poly1305 is an AEAD algorithm, that combines the ChaCha20 stream cipher with the Poly1305 message authentication code. It has fast software performance, and without hardware acceleration, is usually faster than AES-GCM.

+
+ + +
+

Public-key cryptography / asymmetric cryptography

+

Asymmetric cryptography makes use pairs of related keys. Each key pair consists of a public key and a corresponding private key. Key pairs are generated with cryptographic algorithms based on mathematical problems termed one-way functions. Security of public-key cryptography depends on keeping the private key secret, while the public key can be openly distributed without compromising security.

+

Usually we use public key cryptography for:

+
    +
  • public key encryption - a message is encrypted with the intended recipient's public key. For properly chosen and used algorithms, messages cannot in practice be decrypted by anyone who does not possess the matching private key, who is thus presumed to be the owner of that key and so the person associated with the public key. This can be used to ensure confidentiality of a message.
    • +
  • digital signatures - a message is signed with the sender's private key and can be verified by anyone who has access to the sender's public key. This verification proves that the sender had access to the private key, and therefore is very likely to be the person associated with the public key. It also proves that the signature was prepared for that exact message, since a signature that passes verification with the public key on one message will not pass verification with the public key on other messages.
    • +
+
+ + +
+

RSA

+ + + + + + + + + + + +
DesignersRon Rivest, Adi Shamir and Leonard Adleman
Published1977, patented until 2000 :(
+
+

Using RSA

+

The security of RSA relies on the practical difficulty of factoring the product of two large prime numbers, the "factoring problem". Still the most widely used public key system. A key size of 2048 or 4096 bits should be used. A very good CSPRNG is needed.

+

In modern cryptographic systems and protocols (like TLS >= 1.2) RSA is only used for verifying that the client is initiating session with the "right" server. Encryption / decryption is performed using common negotiated key and symmetric ciphers.

+

Forward secrecy is a desired feature of specific key-agreement protocols that gives assurances that session keys will not be compromised even if long-term secrets used in the session key exchange are compromised, limiting damage. For HTTPS, the long-term secret is typically the private key of the server.

+
+ + +
+

Post-quantum cryptography

+

Post-quantum cryptography (PQC), sometimes referred to as quantum-proof, quantum-safe, or quantum-resistant, is the development of cryptographic algorithms (usually public-key algorithms) that are thought to be secure against a cryptanalytic attack by a quantum computer. Most widely-used public-key algorithms rely on the difficulty of one of three mathematical problems: the integer factorization problem, the discrete logarithm problem or the elliptic-curve discrete logarithm problem. All of these problems could be easily solved on a sufficiently powerful quantum computer running Shor's algorithm or even faster and less demanding (in terms of the number of qubits required) alternatives.

+
+ + +
+

Q & A

+
+ + +
+
+ + + + + + + + + + diff --git a/reveal.js/demo.html b/reveal.js/demo.html index 39b014d..20ed6cd 100644 --- a/reveal.js/demo.html +++ b/reveal.js/demo.html @@ -1,47 +1,58 @@ - + - - + reveal.js – The HTML Presentation Framework - - + + - - + + - + - - - + + + - + -
-
- +

The HTML Presentation Framework

- Created by Hakim El Hattab and contributors + Created by Hakim El Hattab and + contributors

Hello There

- reveal.js enables you to create beautiful interactive slide decks using HTML. This presentation will show you examples of what it can do. + reveal.js enables you to create beautiful interactive slide decks using HTML. This + presentation will show you examples of what it can do.

@@ -51,21 +62,38 @@

Vertical Slides

Slides can be nested inside of each other.

Use the Space key to navigate through all slides.

-
+
- Down arrow + Down arrow

Basement Level 1

-

Nested slides are useful for adding additional detail underneath a high level horizontal slide.

+

+ Nested slides are useful for adding additional detail underneath a high level + horizontal slide. +

Basement Level 2

That's it, time to go back up.

-
+
- Up arrow + Up arrow
@@ -73,37 +101,46 @@

Slides

- Not a coder? Not a problem. There's a fully-featured visual editor for authoring these, try it out at https://slides.com. + Not a coder? Not a problem. There's a fully-featured visual editor for authoring these, + try it out at https://slides.com.

Hidden Slides

- This slide is visible in the source, but hidden when the presentation is viewed. You can show all hidden slides by setting the `showHiddenSlides` config option to `true`. + This slide is visible in the source, but hidden when the presentation is viewed. You can + show all hidden slides by setting the `showHiddenSlides` config option to `true`.

Pretty Code


-						import React, { useState } from 'react';
+						import { useState } from 'react';
 
 						function Example() {
 						  const [count, setCount] = useState(0);
 
 						  return (
-						    ...
+
+							  ...
+
 						  );
 						}
-

Code syntax highlighting courtesy of highlight.js.

+

+ Code syntax highlighting courtesy of + highlight.js. +

-

With animations

-
+
+

Lightbox

+ Turn any element into a lightbox using data‑preview‑image & data‑preview‑video. +
+
+ 

+								<img src="image.png" data-preview-image="image.png">
+
+ +
+
+ 

+								<img src="video.png" data-preview-video="video.mp4">
+
+ +
+
+
+

Add the r-fit-text class to auto-size text

FIT TEXT

@@ -207,7 +295,10 @@

Fragments

Hit the next arrow...

... to step through ...

-

... a fragmented slide.

+

+ ... a fragmented + slide. +

Transition Styles

- You can select from different transitions, like:
+ You can select from different transitions, like:
None - Fade - Slide - @@ -247,19 +342,73 @@

Themes

- reveal.js comes with a few themes built in:
+ reveal.js comes with a few themes built in:
- Black (default) - - White - - League - - Sky - - Beige - - Simple
- Serif - - Blood - - Night - - Moon - - Solarized + Black (default) + - + White + - + League + - + Sky + - + Beige + - + Simple +
+ Serif + - + Blood + - + Night + - + Moon + - + Solarized

@@ -267,10 +416,18 @@

Slide Backgrounds

- Set data-background="#dddddd" on a slide to change the background color. All CSS color formats are supported. + Set data-background="#dddddd" on a slide to change the background color. + All CSS color formats are supported.

- Down arrow + Down arrow
@@ -282,12 +439,19 @@

Image Backgrounds

<section data-background="image.png">
-
+

Tiled Backgrounds

<section data-background="image.png" data-background-repeat="repeat" data-background-size="100px">
-
-
+
+

Video Backgrounds

<section data-background-video="video.mp4,video.webm">
@@ -297,26 +461,48 @@
-
+

Background Transitions

- Different background transitions are available via the backgroundTransition option. This one's called "zoom". + Different background transitions are available via the backgroundTransition option. This + one's called "zoom". 

Reveal.configure({ backgroundTransition: 'zoom' })
-
+

Background Transitions

-

- You can override background transitions per-slide. -

+

You can override background transitions per-slide.

<section data-background-transition="zoom">
-
+

Iframe Backgrounds

-

Since reveal.js runs on the web, you can easily embed other web content. Try interacting with the page in the background.

+

+ Since reveal.js runs on the web, you can easily embed other web content. Try + interacting with the page in the background. +

@@ -372,11 +558,19 @@

Clever Quotes

- These guys come in two forms, inline: The nice thing about standards is that there are so many to choose from and block: + These guys come in two forms, inline: + The nice thing about standards is that there are so many to choose from + and block:

-
- “For years there has been a theory that millions of monkeys typing at random on millions of typewriters would - reproduce the entire works of Shakespeare. The Internet has proven this theory to be untrue.” +
+ “For years there has been a theory that millions of monkeys typing at random on + millions of typewriters would reproduce the entire works of Shakespeare. The Internet + has proven this theory to be untrue.”
@@ -390,18 +584,36 @@

Speaker View

-

There's a speaker view. It includes a timer, preview of the upcoming slide as well as your speaker notes.

+

+ There's a speaker view. It includes a + timer, preview of the upcoming slide as well as your speaker notes. +

Press the S key to try it out.

Export to PDF

-

Presentations can be exported to PDF, here's an example:

- +

+ Presentations can be exported to PDF, + here's an example: +

+
@@ -416,7 +628,8 @@

State Events

- Additionally custom events can be triggered on a per slide basis by binding to the data-state name. + Additionally custom events can be triggered on a per slide basis by binding to the + data-state name. 


 Reveal.on( 'customevent', function() {
@@ -428,7 +641,8 @@ Reveal.on( 'customevent', function() {
 				

 					
Take a Moment

 					

-						Press B or . on your keyboard to pause the presentation. This is helpful when you're on stage and want to take distracting slides off the screen.
+						Press B or . on your keyboard to pause the presentation. This is helpful when you're on
+						stage and want to take distracting slides off the screen.
 					

 				

 
@@ -438,33 +652,40 @@ Reveal.on( 'customevent', function() {
 						
  • Right-to-left support
    • 
 						
  • Extensive JavaScript API
    • 
 						
  • Auto-progression
    • 
-						
  • Parallax backgrounds
    • 
+						
  • 
+							Parallax backgrounds
+						
    • 
 						
  • Custom keyboard bindings
    •
    -
    +

    THE END

    - - Try the online editor
    + - Try the online editor
    - Source code & documentation

    -
    -
    - - - - - + + + + + - diff --git a/reveal.js/dist/theme/statnett_compact.css b/reveal.js/dist/theme/statnett_compact.css new file mode 100644 index 0000000..b9f8dd8 --- /dev/null +++ b/reveal.js/dist/theme/statnett_compact.css @@ -0,0 +1,307 @@ +/** + * A simple theme for reveal.js presentations, similar + * to the default theme. The accent color is brown. + * + * This theme is Copyright (C) 2012-2013 Owen Versteeg, http://owenversteeg.com - it is MIT licensed. + */ +.reveal a { + line-height: 1.3em; } + +section.has-dark-background, section.has-dark-background h1, section.has-dark-background h2, section.has-dark-background h3, section.has-dark-background h4, section.has-dark-background h5, section.has-dark-background h6 { + color: #fff; } + +/********************************************* + * GLOBAL STYLES + *********************************************/ +:root { + --background-color: #F0F1EB; + --main-font: Palatino Linotype, Book Antiqua, Palatino, FreeSerif, serif; + --main-font-size: 40px; + --main-color: #000; + --block-margin: 20px; + --heading-margin: 0 0 20px 0; + --heading-font: Palatino Linotype, Book Antiqua, Palatino, FreeSerif, serif; + --heading-color: #383D3D; + --heading-line-height: 1.2; + --heading-letter-spacing: normal; + --heading-text-transform: none; + --heading-text-shadow: none; + --heading-font-weight: normal; + --heading1-text-shadow: none; + --heading1-size: 3.77em; + --heading2-size: 2.11em; + --heading3-size: 1.55em; + --heading4-size: 1em; + --code-font: monospace; + --link-color: #51483D; + --link-color-hover: #8b7c69; + --selection-background-color: #26351C; + --selection-color: #fff; } + +.reveal-viewport { + background: #F0F1EB; + background-image: url("statnett.svg"); + background-repeat: no-repeat; + background-color: #F0F1EB; } + +.reveal { + font-family: "Palatino Linotype", "Book Antiqua", Palatino, FreeSerif, serif; + /* font-size: 24px; */ + font-size: 20px; + font-weight: normal; + color: #000; } + +.reveal ::selection { + color: #fff; + background: #26351C; + text-shadow: none; } + +.reveal ::-moz-selection { + color: #fff; + background: #26351C; + text-shadow: none; } + +.reveal .slides section, +.reveal .slides section > section { + line-height: 1.3; + font-weight: inherit; } + +/********************************************* + * HEADERS + *********************************************/ +.reveal h1, +.reveal h2, +.reveal h3, +.reveal h4, +.reveal h5, +.reveal h6 { + /* margin: 0 0 20px 0; */ + margin: 0 0 16px 0; + color: #383D3D; + font-family: "Palatino Linotype", "Book Antiqua", Palatino, FreeSerif, serif; + font-weight: normal; + line-height: 1.2; + letter-spacing: normal; + text-transform: none; + text-shadow: none; + word-wrap: break-word; } + +.reveal h1 { + font-size: 3.77em; } + +.reveal h2 { + font-size: 2.11em; } + +.reveal h3 { + font-size: 1.55em; } + +.reveal h4 { + font-size: 1em; } + +.reveal h1 { + text-shadow: none; } + +/********************************************* + * OTHER + *********************************************/ +.reveal p { + /* margin: 20px 0; */ + margin: 16px 0; + line-height: 1.3; } + +/* Remove trailing margins after titles */ +.reveal h1:last-child, +.reveal h2:last-child, +.reveal h3:last-child, +.reveal h4:last-child, +.reveal h5:last-child, +.reveal h6:last-child { + margin-bottom: 0; } + +/* Ensure certain elements are never larger than the slide itself */ +.reveal img, +.reveal video, +.reveal iframe { + max-width: 95%; + max-height: 95%; } + +.reveal strong, +.reveal b { + font-weight: bold; } + +.reveal em { + font-style: italic; } + +.reveal ol, +.reveal dl, +.reveal ul { + display: inline-block; + text-align: left; + margin: 0 0 0 1em; } + +.reveal ol { + list-style-type: decimal; } + +.reveal ul { + list-style-type: disc; } + +.reveal ul ul { + list-style-type: square; } + +.reveal ul ul ul { + list-style-type: circle; } + +.reveal ul ul, +.reveal ul ol, +.reveal ol ol, +.reveal ol ul { + display: block; + /* margin-left: 40px; } */ + margin-left: 34px; } + +.reveal dt { + font-weight: bold; } + +.reveal dd { + margin-left: 40px; } + +.reveal blockquote { + display: block; + position: relative; + width: 70%; + margin: 20px auto; + padding: 5px; + font-style: italic; + background: rgba(255, 255, 255, 0.05); + box-shadow: 0px 0px 2px rgba(0, 0, 0, 0.2); } + +.reveal blockquote p:first-child, +.reveal blockquote p:last-child { + display: inline-block; } + +.reveal q { + font-style: italic; } + +.reveal pre { + display: block; + position: relative; + width: 90%; + margin: 20px auto; + text-align: left; + font-size: 0.55em; + font-family: monospace; + line-height: 1.1em; + word-wrap: break-word; } + /* box-shadow: 0px 5px 15px rgba(0, 0, 0, 0.15); } */ + +.reveal code { + font-family: monospace; + text-transform: none; } + +.reveal pre code { + display: block; + padding: 4px; + overflow: auto; + max-height: 520px; + word-wrap: normal; } + +.reveal table { + margin: auto; + border-collapse: collapse; + border-spacing: 0; } + +.reveal table th { + font-weight: bold; } + +.reveal table th, +.reveal table td { + text-align: left; + padding: 0.2em 0.5em 0.2em 0.5em; + border-bottom: 1px solid; } + +.reveal table th[align="center"], +.reveal table td[align="center"] { + text-align: center; } + +.reveal table th[align="right"], +.reveal table td[align="right"] { + text-align: right; } + +.reveal table tbody tr:last-child th, +.reveal table tbody tr:last-child td { + border-bottom: none; } + +.reveal sup { + vertical-align: super; + font-size: smaller; } + +.reveal sub { + vertical-align: sub; + font-size: smaller; } + +.reveal small { + display: inline-block; + font-size: 0.6em; + line-height: 1.2em; + vertical-align: top; } + +.reveal small * { + vertical-align: top; } + +.reveal img { + margin: 20px 0; } + +/********************************************* + * LINKS + *********************************************/ +.reveal a { + color: #51483D; + text-decoration: none; + transition: color .15s ease; } + +.reveal a:hover { + color: #8b7c69; + text-shadow: none; + border: none; } + +.reveal .roll span:after { + color: #fff; + background: #25211c; } + +/********************************************* + * Frame helper + *********************************************/ +.reveal .r-frame { + border: 4px solid #000; + box-shadow: 0 0 10px rgba(0, 0, 0, 0.15); } + +.reveal a .r-frame { + transition: all .15s linear; } + +.reveal a:hover .r-frame { + border-color: #51483D; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.55); } + +/********************************************* + * NAVIGATION CONTROLS + *********************************************/ +.reveal .controls { + color: #51483D; } + +/********************************************* + * PROGRESS BAR + *********************************************/ +.reveal .progress { + background: rgba(0, 0, 0, 0.2); + color: #51483D; } + +#smallertext { + font-size: 0.48em; +} + +/********************************************* + * PRINT BACKGROUND + *********************************************/ +@media print { + .backgrounds { + background-color: #F0F1EB; } } diff --git a/reveal.js/free_software.html b/reveal.js/free_software.html index 71bea85..7421378 100644 --- a/reveal.js/free_software.html +++ b/reveal.js/free_software.html @@ -1,25 +1,25 @@ - + - + Free software in a nutshell - - - - + + + + - - + + - - + + +
    -
    @@ -192,28 +192,30 @@ copies or substantial portions of the Software."

    - - - - - - - - + + + + + + + diff --git a/reveal.js/git.html b/reveal.js/git.html old mode 100755 new mode 100644 index 6ee3406..69e658a --- a/reveal.js/git.html +++ b/reveal.js/git.html @@ -1,25 +1,26 @@ - + - + Introduction to git - - - - + + + + - - + + - + + - - + + +
    -
    @@ -486,28 +487,30 @@
    - - - - - - - - + + + + + + + diff --git a/reveal.js/images/cryptographic_primitives/GCM.svg b/reveal.js/images/cryptographic_primitives/GCM.svg new file mode 100644 index 0000000..5d59d8c --- /dev/null +++ b/reveal.js/images/cryptographic_primitives/GCM.svg @@ -0,0 +1,1500 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/reveal.js/images/cryptographic_primitives/hmac.svg b/reveal.js/images/cryptographic_primitives/hmac.svg new file mode 100644 index 0000000..9f6d2f8 --- /dev/null +++ b/reveal.js/images/cryptographic_primitives/hmac.svg @@ -0,0 +1,165 @@ + +{\displaystyle {\begin{aligned}\operatorname {HMAC} (K,m)&=\operatorname {H} {\Bigl (}{\bigl (}K'\oplus opad{\bigr )}\parallel \operatorname {H} {\bigl (}\left(K'\oplus ipad\right)\parallel m{\bigr )}{\Bigr )}\\K'&={\begin{cases}\operatorname {H} \left(K\right)&{\text{if}}\ K{\text{ is larger than block size}}\\K&{\text{otherwise}}\end{cases}}\end{aligned}}} + + + \ No newline at end of file diff --git a/reveal.js/images/cryptographic_primitives/modes_of_operation.png b/reveal.js/images/cryptographic_primitives/modes_of_operation.png new file mode 100644 index 0000000..d7c9359 Binary files /dev/null and b/reveal.js/images/cryptographic_primitives/modes_of_operation.png differ diff --git a/reveal.js/images/dataporten/dataporten_the_big_picture1.png b/reveal.js/images/dataporten/dataporten_the_big_picture1.png deleted file mode 100644 index 436e2a5..0000000 Binary files a/reveal.js/images/dataporten/dataporten_the_big_picture1.png and /dev/null differ diff --git a/reveal.js/images/dataporten/flow1.png b/reveal.js/images/dataporten/flow1.png deleted file mode 100644 index 0ae3823..0000000 Binary files a/reveal.js/images/dataporten/flow1.png and /dev/null differ diff --git a/reveal.js/images/gateway/dashboard01.png b/reveal.js/images/gateway/dashboard01.png deleted file mode 100644 index 08db6b3..0000000 Binary files a/reveal.js/images/gateway/dashboard01.png and /dev/null differ diff --git a/reveal.js/images/gateway/portal01.png b/reveal.js/images/gateway/portal01.png deleted file mode 100644 index 8ffcad5..0000000 Binary files a/reveal.js/images/gateway/portal01.png and /dev/null differ diff --git a/reveal.js/images/gateway/portal02.png b/reveal.js/images/gateway/portal02.png deleted file mode 100644 index f60e742..0000000 Binary files a/reveal.js/images/gateway/portal02.png and /dev/null differ diff --git a/reveal.js/images/mqprod/flow01.png b/reveal.js/images/mqprod/flow01.png deleted file mode 100644 index 28547fb..0000000 Binary files a/reveal.js/images/mqprod/flow01.png and /dev/null differ diff --git a/reveal.js/mqprod.html b/reveal.js/mqprod.html deleted file mode 100644 index 9aef4bc..0000000 --- a/reveal.js/mqprod.html +++ /dev/null @@ -1,248 +0,0 @@ - - - - - Meldingskø for UiO - - - - - - - - - - - - - - -
    - - -
    -
    -

    Meldingskø for UiO

    -

    Utviklerforum 14.12.2017

    -
    -

    - Kai Vaade (KIA), Simeon Simeonov (INT) -

    -
    - -
    -

    Agenda

    -
    -
      -
    • Administrasjon
      • -
    • Roller og entiteter
      • -
    • Flyt
      • -
    • Rettigheter
      • -
    • Q & A
      • -
    -
    - -
    -

    Generelt

    -
    -

    Vi bruker RabbitMQ med AMQP 0.9.1 til å utveksle / behandle meldinger (JSON, SCIM)

    -

    Bruk av message-broker startet i 2015. I produksjon siden høsten 2017.

    -

    Driftes av KIA.

    -
    - -
    -

    Administrasjon

    -

    Cerebrum har brukt MQ et års tid på egen server satt opp av Seksjon for integrasjon og elektroniske identiteter (USITINT).

    -

    Nå er driften av tjenesten flyttet fra prosjektet og inn i linja, Gruppe for drift av katalog-, integrasjon- og autentiseringstjenester (KIA).

    -
    - -
    -

    Administrasjon (forts...)

    -

    Vi har softlaunchet meldingskø-tjenesten i høst. Tjenesten er ikke "offisielt" lansert, kjører kun våre egne ting (Cerebrum/SAP) foreløpig.

    -

    Antakelig gjør vi tjenesten kjent samtidig som et annet delprosjekt (API Manager) i UiO INTARK lanseres i nær fremtid.

    -
    - -
    -

    Administrasjon (forts...)

    -

    Foreløpig så har vi ikke så veldig mye erfaring med tjenesten i linja.

    -

    Vi vet heller ikke så mye om behovet; dvs hvor mange som kommer til å bruke tjenesten.

    -
    - -
    -

    Administrasjon (forts...)

    -

    Hvordan kontakte oss:

    -

    https://www.usit.uio.no/om/organisasjon/iti/td/kia/dokumentasjon/meldingsko/

    -

    Tilgang til grensesnittet: https://mq.uio.no

    -
    - -
    -

    Roller og entiteter

    -
      -

      Roller:

      -
    • Administrator / Manager
      • -
    • Consumer / Konsument
      • -
    • Publisher / Publisist :)
      • -

      Entiteter:

      -
    • vhost
      • -
    • Exchange (direct, fanout, topic, ...)
      • -
    • Kø (transient, durable)
      • -
    • Binding
      • -
    -
    - -
    -

    Flyt

    -
    - -
    - -
    -

    Rettigheter

    -

    RabbitMQ implementerer 2 rettighetsnivåer: per vhost og per entitet

    -

    RabbitMQ (AMQP) definerer 3 typer operasjoner:

    -
      -

    • configure - opprette / slette entiteter eller endre deres oppførsel

      • -

    • write - skrive melding til en entitet

      • -

    • read - lese melding fra entitet

      • -
    -
    - -
    -

    Rettigheter og brukere

    -

    RabbitMQ bruker regular expressions til å definere rettigheter

    -
      -

    • rabbitmqctl add_user cerebrum <passord>

      • -

    • rabbitmqctl add_user uio_ad_microservice <passord>

      • -

    • rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"

      • -

    • rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"

      • -
    -
    - -
    -

    Routing keys i topic exchange

    -

    Routing key settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en topic exchange.

    -

    Strukturen til en topic / message routing key er:

    -

    <kilde>.<type>.<objekt>.<hendelse>

    -

    F.eks.:

    -

    cerebrum.event.person.delete

    -
    - -
    -

    Dokumentasjon og lenker

    -

    - RabbitMQ: -

    - -
    - -
    -

    Dokumentasjon og lenker

    -

    - UiO: -

    - -

    - Bøker: -

    -
      -
    • -

      - - RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning - -

      • -
    • -

      - - Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing - -

      • -
    - -
    -

    Q & A

    -
    - -
    -
    - - - - - - - - - - - diff --git a/reveal.js/postgresql_tuning.html b/reveal.js/postgresql_tuning.html index 8678513..fb316ff 100644 --- a/reveal.js/postgresql_tuning.html +++ b/reveal.js/postgresql_tuning.html @@ -1,151 +1,155 @@ - + - - - SQLAlchemy - - - - - - - - - - - - - - - -
    - - -
    - -
    -

    PostgreSQL tuning

    -

    Data Science @ Beryl

    + + + Introduction to cryptographic primitives + + + + + + + + + + + + + + + + + +
    + +
    + +
    +

    PostgreSQL tuning

    +

    Data Science @ Beryl


    -

    Simeon Simeonov

    -
    +

    Simeon Simeonov

    +
    -
    +
    -
    -

    Agenda

    +
    +

    Agenda


      -
    • Generic tools for gathering information
      •  -
    • Memory settings
      •  -
    • Logging and performance reports
      •  -
    • Other tools for analysis
      •  +
    • Generic tools for gathering information
      •  +
    • Memory settings
      •  +
    • Logging and performance reports
      •  +
    • Other tools for analysis
      •
    -
    +
    -
    +
    -
    -

    General tools for gathering information

    +
    +

    General tools for gathering information


    -              
-                  # shell
-                  # fetch information from the OS
-                  cat /proc/cpuinfo
-                  cat /proc/meminfo
-                  sysctl -a | grep shm  # get kernel parameters of interest
-              
-              
-                  -- SQL
-                  -- show the current values of all settings
-                  SHOW ALL;
-
-                  -- display even more than all...
-                  SELECT * FROM pg_settings;
-
-                  -- opening postgresql.conf and reading the comments - the old school approach
-              
-              
-                  # old school: edit postgresql.conf and read the comments
-              
+            
+              # shell
+              # fetch information from the OS
+              cat /proc/cpuinfo
+              cat /proc/meminfo
+              sysctl -a | grep shm  # get kernel parameters of interest
+            
+            
+              -- SQL
+              -- show the current values of all settings
+              SHOW ALL;
+
+              -- display even more than all...
+              SELECT * FROM pg_settings;
+
+              -- opening postgresql.conf and reading the comments - the old school approach
+            
+            
+              # old school: edit postgresql.conf and read the comments
+
    -
    +
    -
    -

    Memory settings

    +
    +

    Memory settings


      -
    • shared_buffers - how much memory is dedicated to PostgreSQL to use for caching data - for a system with 1GB or more of RAM, a reasonable starting value for shared_buffers is 25% of the system memory (128MB -> 1GB)
      • -
    • effective_cache_size - how much memory we expect to be available in the OS and PostgreSQL buffer caches, not an allocation - used only by the PostgreSQL query planner to figure out whether plans it's considering would be expected to fit in RAM or not - 1/2 of total memory would be a normal conservative setting (4GB -> 8GB)
      • -
    • work_mem - the base maximum amount of memory to be used by a query operation (such as a sort or hash table) before writing to temporary disk files - for a complex query, several sort or hash operations might be running in parallel; each operation will generally be allowed to use as much memory as this value specifies (4MB -> 8MB)
      • -
    • maintenance_work_mem - the maximum amount of memory to be used by maintenance operations, such as VACUUM and CREATE INDEX. It's safe to set this value significantly larger than work_mem (64MB -> 256MB)
      • +
    • shared_buffers - how much memory is dedicated to PostgreSQL to use for caching data - for a system with 1GB or more of RAM, a reasonable starting value for shared_buffers is 25% of the system memory (128MB -> 1GB)
      • +
    • effective_cache_size - how much memory we expect to be available in the OS and PostgreSQL buffer caches, not an allocation - used only by the PostgreSQL query planner to figure out whether plans it's considering would be expected to fit in RAM or not - 1/2 of total memory would be a normal conservative setting (4GB -> 8GB)
      • +
    • work_mem - the base maximum amount of memory to be used by a query operation (such as a sort or hash table) before writing to temporary disk files - for a complex query, several sort or hash operations might be running in parallel; each operation will generally be allowed to use as much memory as this value specifies (4MB -> 8MB)
      • +
    • maintenance_work_mem - the maximum amount of memory to be used by maintenance operations, such as VACUUM and CREATE INDEX. It's safe to set this value significantly larger than work_mem (64MB -> 256MB)
    -
    +
    -
    -

    Logging and performance reports

    +
    +

    Logging and performance reports


    -

    pgBadger - a fast PostgreSQL log analysis report can be used for general analysis.

    -
      -
    • log_checkpoints - checkpoints and restartpoints are logged in the server log. Some statistics are included in the log messages, including the number of buffers written and the time spent writing them
      • -
    • log_connections - each attempted connection to the server to be logged, as well as successful completion of client authentication
      • -
    • log_disconnections - provides information similar to log_connections, plus the duration of the session
      • -
    • log_line_prefix - set to '%t [%p]: user=%u,db=%d,app=%a,client=%h '
      • -
    • log_lock_waits - log message is produced when a session waits longer than deadlock_timeout to acquire a lock. This is useful in determining if lock waits are causing poor performance
      • -
    • log_temp_files - when set to 0, a log entry is emitted for each temporary file when it is deleted
      • -
    • log_autovacuum_min_duration - set to 0 it logs all autovacuum actions
      • -
    -
    - -
    -

    Other tools for analysis

    +

    pgBadger - a fast PostgreSQL log analysis report can be used for general analysis.

    +
      +
    • log_checkpoints - checkpoints and restartpoints are logged in the server log. Some statistics are included in the log messages, including the number of buffers written and the time spent writing them
      • +
    • log_connections - each attempted connection to the server to be logged, as well as successful completion of client authentication
      • +
    • log_disconnections - provides information similar to log_connections, plus the duration of the session
      • +
    • log_line_prefix - set to '%t [%p]: user=%u,db=%d,app=%a,client=%h '
      • +
    • log_lock_waits - log message is produced when a session waits longer than deadlock_timeout to acquire a lock. This is useful in determining if lock waits are causing poor performance
      • +
    • log_temp_files - when set to 0, a log entry is emitted for each temporary file when it is deleted
      • +
    • log_autovacuum_min_duration - set to 0 it logs all autovacuum actions
      • +
    +
    + +
    +

    Other tools for analysis


      -
    • ANALYZE - collects statistics about the contents of tables in the database, and stores the results in the pg_statistic system catalog. Subsequently, the query planner uses these statistics to help determine the most efficient execution plans for queries.
      • -
    • VACUUM - reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. (VACUUM vs. VACUUM FULL)
      • +
    • ANALYZE - collects statistics about the contents of tables in the database, and stores the results in the pg_statistic system catalog. Subsequently, the query planner uses these statistics to help determine the most efficient execution plans for queries.
      • +
    • VACUUM - reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. (VACUUM vs. VACUUM FULL)
    -
    +
    -
    -

    Sources

    +
    +

    Sources


    -

    - https://www.postgresql.org/docs/ - The official documentation -

    -

    - https://wiki.postgresql.org - The official Wiki -

    -
    - -
    -

    Q & A

    -
    - -
    -
    - - - - - - - - - - +

    + https://www.postgresql.org/docs/ - The official documentation +

    +

    + https://wiki.postgresql.org - The official Wiki +

    + + +
    +

    Q & A

    +
    + + + + + + + + + + + + diff --git a/reveal.js/rabbitmq.html b/reveal.js/rabbitmq.html deleted file mode 100644 index 6d1ba7a..0000000 --- a/reveal.js/rabbitmq.html +++ /dev/null @@ -1,311 +0,0 @@ - - - - - RabbitMQ - Intern opplæring - - - - - - - - - - - - - - -
    - - -
    -
    -

    RabbitMQ

    -

    Intern opplæring

    -
    -

    - Simeon Simeonov -

    -
    - -
    -
    -

    Agenda

    -
    -
      -
    • Installasjon
      •  -
    • Administrasjon
      •  -
    • Utvikling
      •  -
    • Diverse + Q&A
      •  -
    -
    -
    - -
    -
    -

    Installasjon og oppsett

    -
    -
      -

    • Skaffe seg maskin og SSL sertifikat(er)

      • -

    • Bruke Ansible

      • -

    • Opprette administrator-bruker og slette guest-brukeren

      • -

    • Sette rettigheter og opprette entiteter

      • -
    -
    -
    -

    Ansible

    -
    -

    Vi bruker repo UAIT/int-ansible-hosts

    -

    Etter å ha installert Ansible på lokalmaskina gjør vi følgende:

    -
      -

    • git clone ssh://git@bitbucket.usit.uio.no:7999/uait/int-ansible-hosts.git

      • -

    • cd int-ansible-hosts

      • -

    • ansible-playbook --ask-become-pass -v --extra-vars '{"hosts": "mq-hostname"}' rabbitmq_playbook.yml

      • -
    -

    RabbitMQ skal nå være installert og tilgjengelig på https://mq-hostname

    -
    - -
    -

    Administrator-bruker

    -
    -

    Brukeren guest med passord guest vil eksistere etter at Ansible har kjørt

    -

    I tillegg til webgrensesnittet, kan en bruke rabbitmqctl

    -
      -

    • rabbitmqctl list_users (Lister alle eksisterende RabbitMQ-brukere)

      • -

    • rabbitmqctl add_user rmq_admin <passord> (Legger til dedikert admin bruker rmq_admin)

      • -

    • rabbitmqctl set_user_tags rmq_admin administrator (Gir administrator rolle til rmq_admin)

      • -

    • rabbitmqctl delete_user guest (Sletter brukeren guest)

      • -
    -
    - -
    -

    vhost

    -
    -

    vhost fungerer som en beholder (container) for RabbitMQ/AMQP objekter. Nok igjen kan en velge mellom webgrensesnitt og rabbitmqctl

    -
      -

    • rabbitmqctl list_vhosts (Lister alle eksisterende vhosts)

      • -

    • rabbitmqctl add_vhost /no/uio/integration (Oppretter vhost med navn /no/uio/integration)

      • -
    -
    - -
    -

    Rettigheter

    -

    RabbitMQ implementerer 2 rettighetsnivåer(*):

    -
      -
    • Per vhost
      • -
    • Per entitet
      • -
    -

    RabbitMQ definerer 3 typer operasjoner:

    -
      -

    • configure - opprette / slette entiteter eller endre deres oppførsel

      • -

    • write - skrive melding til en entitet

      • -

    • read - lese melding fra entitet

      • -
    -

    (*) - /etc/rabbitmq/rabbitmq.config gir flere muligheter

    -
    - -
    -

    Rettigheter og brukere

    -

    RabbitMQ bruker regular expressions til å definere rettigheter

    -
      -

    • rabbitmqctl add_user cerebrum <passord>

      • -

    • rabbitmqctl add_user uio_ad_microservice <passord>

      • -

    • rabbitmqctl set_permissions -p /no/uio/integration cerebrum "^$" "^ex_.*" "^$"

      • -

    • rabbitmqctl set_permissions -p /no/uio/integration uio_ad_microservice "^q_ad_ms_.*" "^q_ad_ms_.*" "^(ex_messages|q_ad_ms_.*)$"

      • -
    -
    - -
    -

    Køer

    -

    Køer defineres som durable ved hjelp av webgrensesnittet.

    -

    "Alle køer (både forhåndsdefinerte og de som defineres av konsument) må ha navn som starter med q_. Køen vil ha navn som uttryker mottakeren som bruker den."

    -

    F.eks. q_ad_ms_all vil være et passende navn for køen som brukes av AD-microservice mottakeren.

    -
    - -
    -

    Exchange

    -

    "Avsendere vil vanligvis sende meldinger til exchange ex_messages."

    -

    Det er flere typer exchange, men vi vil bruke kun topic exchange.

    -
    -
    - -
    -
    -

    Utvikling

    -
    -
      -

    • Routing keys i topic exchange

      • -

    • Bindinger

      • -

    • Protokoller og porter

      • -

    • Eksempler

      • -
    -
    - -
    -

    Routing keys i topic exchange

    -
    -

    Routing key settes av sender som en del av meldingen og blir inspisert av brokeren dersom meldingen sendes til en topic exchange.

    -

    Strukturen til en topic / message routing key er:

    -

    <kilde>.<type>.<objekt>.<hendelse>

    -

    F.eks.:

    -

    cerebrum.event.person.delete

    -
    - -
    -

    Bindinger

    -

    For at en melding sendt til en topic exchange skal havne i en bestemt kø, må køen være bundet (bound) til exchange.

    -

    I topic exchange brukes routing key (topic) til å avgjøre hvilke av meldingene som blir sendt vil havne i køen som er bundet.

    -

    Dersom vi binder køen q_ad_ms_all til topic exchange ex_messages med binding key cerebrum.event.account.* vil alle meldinger som blir sendt til ex_messages med topic som starter med cerebrum.event.account. havne i q_ad_ms_all.

    -

    En kø kan bli bundet med én eller flere nøkler til en topic exchange.

    -
    - -
    -

    Protokoller og porter

    -
    -

    "AMQP 0.9.1 prioritert siden det er protokollen som mapper best mot RabbitMQs funksjonalitet. Vi har også noe erfaring med bruk av denne."

    -

    RabbitMQ lytter for AMQP0.9.1 på SSL port 5671.

    -
    -

    Andre protokoller og tjenester for RabbitMQ:

    -
      -
    • STOMP - 61614 (SSL)
      • -
    • RabbitMQ Management (webgrensesnitt) - 15671 (SSL)
      • -
    -
    - -
    -

    Eksempler

    -
    - 
    -                
-                    $ pip install pika
-                
-
    -

    Python:

    -

    "Well, we'll not risk another frontal assault. That rabbit's dynamite...."

    -
    - -
    - -
    -
    -

    Dokumentasjon og lenker

    -
    -

    - RabbitMQ: -

    - -
    -
    -

    Dokumentasjon og lenker

    -

    - UiO: -

    - -

    - Bøker: -

    -
      -
    • -

      - - RabbitMQ in action - Alvaro Videla / Jason J.W. Williams - 2012 - Manning - -

      • -
    • -

      - - Mastering RabbitMQ - Ayanoglu / Aytas / Nahum - 2015 - PACKT Publishing - -

      • -
    -
    - -
    -

    Q&A

    -
    - -
    -
    - - - - - - - - - - - diff --git a/reveal.js/sqlalchemy.html b/reveal.js/sqlalchemy.html index a0d6617..5dfae21 100644 --- a/reveal.js/sqlalchemy.html +++ b/reveal.js/sqlalchemy.html @@ -1,477 +1,480 @@ - + - - - SQLAlchemy - - - - - - - - - - - - - - - -
    - - -
    - -
    -

    SQLAlchemy

    -

    Data Engineering @ Statnett

    + + + SQLAlchemy + + + + + + + + + + + + + + + + + +
    + +
    + +
    +

    SQLAlchemy

    +

    Data Engineering @ Statnett


    -

    Simeon Simeonov

    -
    +

    Simeon Simeonov

    +
    -
    +
    -
    -

    Agenda

    +
    +

    Agenda


      -
    • SQLAlchemy - Design & overview
      •  -
    • SQLAlchemy - A small practical example
      •  -
    • Q & A
      •  +
    • SQLAlchemy - Design & overview
      •  +
    • SQLAlchemy - A small practical example
      •  +
    • Q & A
      •
    -
    +
    -
    +
    -
    -

    What is SQLAlchemy?

    +
    +

    What is SQLAlchemy?


    -

    SQLAlchemy is a Python library created by Mike Bayer to provide a high-level Pythonic interface to RDBMS such as PostgreSQL, SQLite, MySQL, Oracle, DB2.

    -

    SQLAlchemy includes RDBMS-independent SQL expression language and an object-relational mapper (ORM).

    -
    +

    SQLAlchemy is a Python library created by Mike Bayer to provide a high-level Pythonic interface to RDBMS such as PostgreSQL, SQLite, MySQL, Oracle, DB2.

    +

    SQLAlchemy includes RDBMS-independent SQL expression language and an object-relational mapper (ORM).

    +
    -
    -

    Why use SQLAlchemy?

    +
    +

    Why use SQLAlchemy?


      -

    • free software - free as in "freedom" (MIT licensed)

      • -

    • portability - the programming interface is independent of the type of RDBMS and connector used

      • -

    • security - no more SQL injections

      • -

    • abstraction - no need to bother with complex JOINs

      • -

    • object-orientation - you work with objects instead of tables and rows

      • -

    • performance - exploits the likehood of reusing a particular query

      • -

    • flexibility - you can override almost anything

      • +

    • free software - free as in "freedom" (MIT licensed)

      • +

    • portability - the programming interface is independent of the type of RDBMS and connector used

      • +

    • security - no more SQL injections

      • +

    • abstraction - no need to bother with complex JOINs

      • +

    • object-orientation - you work with objects instead of tables and rows

      • +

    • performance - exploits the likehood of reusing a particular query

      • +

    • flexibility - you can override almost anything

    -
    - -
    -

    Basic architecture

    -

    SQLAlchemy consists of several components, including the ORM.

    -
      -
    • Engine- manages the connection pool and the RDBMS-independent SQL dialect layer
      • -
    • MetaData - used to collect and organize information about your table layout (schema)
      • -
    • SQL expression language - provides an API to execute your queries and updates against your tables, all from Python, and all in a database-independent way (low-level interface)
      • -
    • ORM - provides a convenient way to add database persistence to your Python objects without requiring you to design your objects around the database, or the database around the objects (high-level interface)
      • -
    • Session - establishes all conversations with the RDBMS and represents a "holding zone" for all the objects which you've loaded or associated with it during its lifespan
      • -
    - -
    - -
    -

    Example

    -

    SQLAlchemy gives us the choice between classical mapping and the newer declarative mapping

    - 
    -                        
-                            # option 1: classical mapping
-                            # explicitly defining Table objects and mapping them to pure Python base classes
-                            from sqlalchemy import create_engine
-
-                            # engine = create_engine("postgresql+psycopg2://user:zipassword@localhost/mydb" , echo=True)
-                            # The string form of the URL is dialect+driver://user:password@host/dbname[?key=value..],
-                            # engine = create_engine("sqlite:///library.db", echo=True)
-                            engine = create_engine("sqlite:///:memory:", echo=True)
-
-                            from sqlalchemy import Column, MetaData, Table
-                            from sqlalchemy import DateTime, ForeignKey, Integer, Numeric, String
-
-                            metadata = MetaData()
-
-                            production_types_table = Table(
-                                "production_types",
-                                metadata,
-                                Column("production_type_id", Integer, primary_key=True),
-                                Column("code", String(3), nullable=False, unique=True),
-                                Column("description", String),  # Column("name", String(128)) is possible
-                            )
-
-                            bidding_areas_table = Table(
-                                "bidding_areas",
-                                metadata,
-                                Column("bidding_area_id", Integer, primary_key=True),
-                                Column("code", String(3), nullable=False, unique=True),
-                                Column("name", String(32)),
-                            )
-
-                            production_plans_table = Table(
-                                "production_plans",
-                                metadata,
-                                Column("record_created_time", DateTime(timezone=False), primary_key=True),
-                                Column("start_time", DateTime(timezone=False), primary_key=True),
-                                Column("bidding_area_id", Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True),
-                                Column("production_type_id", Integer, ForeignKey("production_types.production_type_id"), primary_key=True),
-                                Column("value", Numeric, nullable=False),
-                            )
-
-                            metadata.create_all(engine)  # creates the tables
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Use of SQL expression language

    - 
    -                        
-                            # option 1: classical mapping (continues)
-                            # Using the SQL expression language (low level interface)
-                            from sqlalchemy import text
-
-                            insert_stmt = bidding_areas_table.insert(bind=engine)
-                            type(insert_stmt)
-                            # Out: <class 'sqlalchemy.sql.dml.Insert'>
-                            print(insert_stmt)
-                            # Out: INSERT INTO bidding_areas (bidding_area_id, code, name) VALUES (?, ?, ?)
-
-                            compiled_stmt = insert_stmt.compile()
-                            print(compiled_stmt.params)
-                            # Out: {'bidding_area_id': None, 'code': None, 'name': None}
-
-                            insert_stmt.execute(bidding_area_id=1, code="NO1", name="Elspot NO1")  # insert a single entry
-                            # ... or a list of entries
-                            insert_stmt.execute(
-                                [
-                                    {"bidding_area_id": 2, "code": "NO2", "name": "Elspot NO2"},
-                                    {"bidding_area_id": 3, "code": "NO3", "name": "Elspot NO3"},
-                                    {"bidding_area_id": 4, "code": "NO4", "name": "Elspot NO4"},
-                                    {"bidding_area_id": 5, "code": "NO5", "name": "Elspot NO5"},
-                                    {"bidding_area_id": 6, "code": "NO6", "name": "Elspot NO6"},
-                                ]
-                            )
-
-                            metadata.bind = engine  # no need to explicitly bind the engine from now on
-                            select_stmt = bidding_areas_table.select(bidding_areas_table.c.bidding_area_id==2)
-                            result = select_stmt.execute()
-                            result.fetchall()
-                            # Out: [(2, 'NO2', 'Elspot NO2')]
-
-                            del_stmt = bidding_areas_table.delete()
-                            del_stmt.execute(whereclause=text("name='Elspot NO6'"))
-                            del_stmt.execute()  # delete NO6
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Use of classical mapping

    - 
    -                        
-                            # option 1: classical mapping (continues)
-                            # Defining regular base classes and mapping them to the Table objects
-                            from sqlalchemy.orm import mapper
-
-                            class ProductionType:
-                                def __init__(self, code, description):
-                                    self.code = code
-                                    self.description = description
-
-                                def __str__(self):
-                                    return self.code
-
-
-                            class BiddingArea:
-                                def __init__(self, code, name):
-                                    self.code = code
-                                    self.name = name
-
-                                def __str__(self):
-                                    return self.code
-
-                            mapper(ProductionType, production_types_table)
-                            mapper(BiddingArea, bidding_areas_table)
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Use of classical mapping

    - 
    -                        
-                            from sqlalchemy.orm import relationship
-
-                            class ProductionPlan:
-                                def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
-                                    self.record_created_time = record_created_time
-                                    self.start_time = start_time
-                                    self.production_type = production_type
-                                    self.bidding_area = bidding_area
-                                    self.value = value
-
-                                def __str__(self):
-                                    return (
-                                        f"{self.record_created_time} {self.start_time} "
-                                        f"{self.production_type} {self.bidding_area} {self.value}"
-                                    )
-
-
-                            mapper(
-                                ProductionPlan,
-                                production_plans_table,
-                                properties = {
-                                    "production_type": relationship(ProductionType, backref="production_plans"),
-                                    "bidding_area": relationship(BiddingArea, backref="production_plans"),
-                                },
-                            )
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Doing the same thing the easy way with declarative mapping

    - 
    -                        
-                            # option 2: declarative mapping
-                            from sqlalchemy.ext.declarative import declarative_base
-
-                            Base = declarative_base()
-
-                            class ProductionType(Base):
-                                __tablename__ = "production_types"
-
-                                production_type_id = Column(Integer, primary_key=True)
-                                code = Column(String(3), nullable=False, unique=True)
-                                description = Column(String)
-
-                                def __init__(self, code, description):
-                                    self.code = code
-                                    self.description = description
-
-                                def __str__(self):
-                                    return self.code
-
-                            class BiddingArea(Base):
-                                __tablename__ = "bidding_areas"
-
-                                bidding_area_id = Column(Integer, primary_key=True)
-                                code = Column(String(3), nullable=False, unique=True)
-                                name = Column(String(32))
-
-                                def __init__(self, code, name):
-                                    self.code = code
-                                    self.name = name
-
-                                def __str__(self):
-                                    return self.code
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Doing the same thing the easy way with declarative mapping

    - 
    -                        
-                            # option 2: declarative mapping (continues)
-                            from sqlalchemy.orm import relationship, backref
-
-                            class ProductionPlan(Base):
-                                __tablename__ = "production_plans"
-
-                                record_created_time = Column(DateTime(timezone=False), primary_key=True)
-                                start_time = Column(DateTime(timezone=False), primary_key=True)
-                                bidding_area_id = Column(Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True)
-                                production_type_id = Column(Integer, ForeignKey("production_types.production_type_id"), primary_key=True)
-                                value = Column(Numeric, nullable=False)
-
-                                # defining relationships.
-                                # the defined attributes will reference 'ProductionType' and 'BiddingArea' objects
-                                production_type = relationship(ProductionType, backref=backref("production_plans"))
-                                bidding_area = relationship(BiddingArea, backref=backref("production_plans"))
-
-                                def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
-                                    self.record_created_time = record_created_time
-                                    self.start_time = start_time
-                                    self.production_type = production_type  # a 'ProductionType' object
-                                    self.bidding_area = bidding_area  # a 'BiddingArea' object
-                                    self.value = value
-
-                                def __str__(self):
-                                    return (
-                                        f"{self.record_created_time} {self.start_time} "
-                                        f"{self.production_type} {self.bidding_area} {self.value}"
-                                    )
-
-                            Base.metadata.create_all(engine)  # create tables
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Creating instances

    - 
    -                        
-                            # adding some data...
-                            import datetime
-                            import decimal
-
-                            from sqlalchemy.orm import sessionmaker
-
-                            Session = sessionmaker(bind=engine)  # bound session
-                            session = Session()
-
-                            bidding_area1 = BiddingArea("NO1", "Elspot NO1")
-                            session.add(bidding_area1)
-
-                            session.add_all(
-                                [
-                                    BiddingArea("NO2", "Elspot NO2"),
-                                    BiddingArea("NO3", "Elspot NO3"),
-                                    BiddingArea("NO4", "Elspot NO4"),
-                                    BiddingArea("NO5", "Elspot NO5"),
-                                ]
-                            )
-
-                            production_type_B37 = ProductionType("B37", "Thermal unspecified")
-                            production_type_B30 = ProductionType("B30", "Wind unspecified")
-
-                            session.add_all(
-                                [
-                                    ProductionType("B19", "Wind Onshore"),
-                                    ProductionType("B10", "Hydro-electric pure pumped storage head installation"),
-                                    ProductionType("B11", "Hydro Run-of-river head installation"),
-                                    ProductionType("B12", "Hydro-electric storage head installation"),
-                                    ProductionType("A04", "Generation"),
-                                    production_type_B37,
-                                    production_type_B30,
-                                ]
-                            )
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Creating instances

    - 
    -                        
-                            # adding some production plans...
-
-                            session.add(
-                                ProductionPlan(
-                                    datetime.datetime.now(),
-                                    datetime.datetime(2022, 11, 2, 1, 0),
-                                    production_type_B37,
-                                    bidding_area1,
-                                    decimal.Decimal("80.5"),
-                                )
-                            )
-
-                            production_plan2 = ProductionPlan(
-                                datetime.datetime.now(),
-                                datetime.datetime(2022, 11, 2, 2, 0),
-                                production_type_B37,
-                                bidding_area1,
-                                decimal.Decimal("90.5"),
-                            )
-
-                            session.add(production_plan2)
-
-                            session.flush()  # execute pending operations
-                            session.commit()  # execute and commit pending operations
-
-                            production_plan2.value = decimal.Decimal("70.5")
-                            production_plan2 in session
-                            # Out: True
-
-                            session.commit()
-                        
-
    -
    - -
    -

    Example (cont...)

    -

    Queries

    - 
    -                        
-                            import pandas as pd
-
-                            session.query(ProductionPlan).order_by(ProductionPlan.start_time)  # returns a Query instance
-                            session.query(ProductionPlan).order_by(ProductionPlan.start_time).all()  # returns an object-list
-
-                            # return all production plans where start_time after 2022-09-01 00:00
-                            session.query(ProductionPlan).filter(ProductionPlan.start_time > datetime.datetime(2022, 9, 1, 0, 0)).all()
-
-                            # return production plans with value > 80
-                            query = session.query(ProductionPlan).filter(ProductionPlan.value > 80).order_by(ProductionPlan.start_time)
-                            query.count()  # returns 1
-                            production_plan = query.first()  # returns the first object (element)
-                            production_plan = query.one()  # raises NoResultFound exception or MultipleResultsFound in case elements != 1
-
-                            # generate Pandas DataFrame from a query or entire table
-                            df = pd.read_sql_table("my_table", con=session.get_bind())  # or con=engine
-                            df = pd.read_sql_query(query.statement, engine)
-
-                            # return production plans with production type 'B37'
-                            session.query(ProductionPlan).filter(
-                                ProductionPlan.production_type_id == ProductionType.production_type_id
-                            ).filter(ProductionType.code == "B37").all()
-                            session.query(ProductionPlan).join(ProductionType).filter(ProductionType.code == "B37").all()
-                            session.query(ProductionPlan).filter(ProductionPlan.production_type == production_type_B37).all()
-                            session.query(
-                                ProductionPlan
-                            ).from_statement(
-                                text(
-                                    "SELECT pp.* FROM production_plans pp, production_types pt "
-                                    "WHERE pp.production_type_id = pt.production_type_id AND pt.code=:code"
-                                )
-                            ).params(code="B37").all()
-                        
-
    -
    - -
    -

    Q & A

    -
    - -
    -
    - - - - - - - - - - + + +
    +

    Basic architecture

    +

    SQLAlchemy consists of several components, including the ORM.

    +
      +
    • Engine- manages the connection pool and the RDBMS-independent SQL dialect layer
      • +
    • MetaData - used to collect and organize information about your table layout (schema)
      • +
    • SQL expression language - provides an API to execute your queries and updates against your tables, all from Python, and all in a database-independent way (low-level interface)
      • +
    • ORM - provides a convenient way to add database persistence to your Python objects without requiring you to design your objects around the database, or the database around the objects (high-level interface)
      • +
    • Session - establishes all conversations with the RDBMS and represents a "holding zone" for all the objects which you've loaded or associated with it during its lifespan
      • +
    + +
    + +
    +

    Example

    +

    SQLAlchemy gives us the choice between classical mapping and the newer declarative mapping

    + 
    +            
+              # option 1: classical mapping
+              # explicitly defining Table objects and mapping them to pure Python base classes
+              from sqlalchemy import create_engine
+
+              # engine = create_engine("postgresql+psycopg2://user:zipassword@localhost/mydb" , echo=True)
+              # The string form of the URL is dialect+driver://user:password@host/dbname[?key=value..],
+              # engine = create_engine("sqlite:///library.db", echo=True)
+              engine = create_engine("sqlite:///:memory:", echo=True)
+
+              from sqlalchemy import Column, MetaData, Table
+              from sqlalchemy import DateTime, ForeignKey, Integer, Numeric, String
+
+              metadata = MetaData()
+
+              production_types_table = Table(
+              "production_types",
+              metadata,
+              Column("production_type_id", Integer, primary_key=True),
+              Column("code", String(3), nullable=False, unique=True),
+              Column("description", String),  # Column("name", String(128)) is possible
+              )
+
+              bidding_areas_table = Table(
+              "bidding_areas",
+              metadata,
+              Column("bidding_area_id", Integer, primary_key=True),
+              Column("code", String(3), nullable=False, unique=True),
+              Column("name", String(32)),
+              )
+
+              production_plans_table = Table(
+              "production_plans",
+              metadata,
+              Column("record_created_time", DateTime(timezone=False), primary_key=True),
+              Column("start_time", DateTime(timezone=False), primary_key=True),
+              Column("bidding_area_id", Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True),
+              Column("production_type_id", Integer, ForeignKey("production_types.production_type_id"), primary_key=True),
+              Column("value", Numeric, nullable=False),
+              )
+
+              metadata.create_all(engine)  # creates the tables
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Use of SQL expression language

    + 
    +            
+              # option 1: classical mapping (continues)
+              # Using the SQL expression language (low level interface)
+              from sqlalchemy import text
+
+              insert_stmt = bidding_areas_table.insert(bind=engine)
+              type(insert_stmt)
+              # Out: <class 'sqlalchemy.sql.dml.Insert'>
+              print(insert_stmt)
+              # Out: INSERT INTO bidding_areas (bidding_area_id, code, name) VALUES (?, ?, ?)
+
+              compiled_stmt = insert_stmt.compile()
+              print(compiled_stmt.params)
+              # Out: {'bidding_area_id': None, 'code': None, 'name': None}
+
+              insert_stmt.execute(bidding_area_id=1, code="NO1", name="Elspot NO1")  # insert a single entry
+              # ... or a list of entries
+              insert_stmt.execute(
+              [
+              {"bidding_area_id": 2, "code": "NO2", "name": "Elspot NO2"},
+              {"bidding_area_id": 3, "code": "NO3", "name": "Elspot NO3"},
+              {"bidding_area_id": 4, "code": "NO4", "name": "Elspot NO4"},
+              {"bidding_area_id": 5, "code": "NO5", "name": "Elspot NO5"},
+              {"bidding_area_id": 6, "code": "NO6", "name": "Elspot NO6"},
+              ]
+              )
+
+              metadata.bind = engine  # no need to explicitly bind the engine from now on
+              select_stmt = bidding_areas_table.select(bidding_areas_table.c.bidding_area_id==2)
+              result = select_stmt.execute()
+              result.fetchall()
+              # Out: [(2, 'NO2', 'Elspot NO2')]
+
+              del_stmt = bidding_areas_table.delete()
+              del_stmt.execute(whereclause=text("name='Elspot NO6'"))
+              del_stmt.execute()  # delete NO6
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Use of classical mapping

    + 
    +            
+              # option 1: classical mapping (continues)
+              # Defining regular base classes and mapping them to the Table objects
+              from sqlalchemy.orm import mapper
+
+              class ProductionType:
+              def __init__(self, code, description):
+              self.code = code
+              self.description = description
+
+              def __str__(self):
+              return self.code
+
+
+              class BiddingArea:
+              def __init__(self, code, name):
+              self.code = code
+              self.name = name
+
+              def __str__(self):
+              return self.code
+
+              mapper(ProductionType, production_types_table)
+              mapper(BiddingArea, bidding_areas_table)
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Use of classical mapping

    + 
    +            
+              from sqlalchemy.orm import relationship
+
+              class ProductionPlan:
+              def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
+              self.record_created_time = record_created_time
+              self.start_time = start_time
+              self.production_type = production_type
+              self.bidding_area = bidding_area
+              self.value = value
+
+              def __str__(self):
+              return (
+              f"{self.record_created_time} {self.start_time} "
+              f"{self.production_type} {self.bidding_area} {self.value}"
+              )
+
+
+              mapper(
+              ProductionPlan,
+              production_plans_table,
+              properties = {
+              "production_type": relationship(ProductionType, backref="production_plans"),
+              "bidding_area": relationship(BiddingArea, backref="production_plans"),
+              },
+              )
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Doing the same thing the easy way with declarative mapping

    + 
    +            
+              # option 2: declarative mapping
+              from sqlalchemy.ext.declarative import declarative_base
+
+              Base = declarative_base()
+
+              class ProductionType(Base):
+              __tablename__ = "production_types"
+
+              production_type_id = Column(Integer, primary_key=True)
+              code = Column(String(3), nullable=False, unique=True)
+              description = Column(String)
+
+              def __init__(self, code, description):
+              self.code = code
+              self.description = description
+
+              def __str__(self):
+              return self.code
+
+              class BiddingArea(Base):
+              __tablename__ = "bidding_areas"
+
+              bidding_area_id = Column(Integer, primary_key=True)
+              code = Column(String(3), nullable=False, unique=True)
+              name = Column(String(32))
+
+              def __init__(self, code, name):
+              self.code = code
+              self.name = name
+
+              def __str__(self):
+              return self.code
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Doing the same thing the easy way with declarative mapping

    + 
    +            
+              # option 2: declarative mapping (continues)
+              from sqlalchemy.orm import relationship, backref
+
+              class ProductionPlan(Base):
+              __tablename__ = "production_plans"
+
+              record_created_time = Column(DateTime(timezone=False), primary_key=True)
+              start_time = Column(DateTime(timezone=False), primary_key=True)
+              bidding_area_id = Column(Integer, ForeignKey("bidding_areas.bidding_area_id"), primary_key=True)
+              production_type_id = Column(Integer, ForeignKey("production_types.production_type_id"), primary_key=True)
+              value = Column(Numeric, nullable=False)
+
+              # defining relationships.
+              # the defined attributes will reference 'ProductionType' and 'BiddingArea' objects
+              production_type = relationship(ProductionType, backref=backref("production_plans"))
+              bidding_area = relationship(BiddingArea, backref=backref("production_plans"))
+
+              def __init__(self, record_created_time, start_time, production_type, bidding_area, value):
+              self.record_created_time = record_created_time
+              self.start_time = start_time
+              self.production_type = production_type  # a 'ProductionType' object
+              self.bidding_area = bidding_area  # a 'BiddingArea' object
+              self.value = value
+
+              def __str__(self):
+              return (
+              f"{self.record_created_time} {self.start_time} "
+              f"{self.production_type} {self.bidding_area} {self.value}"
+              )
+
+              Base.metadata.create_all(engine)  # create tables
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Creating instances

    + 
    +            
+              # adding some data...
+              import datetime
+              import decimal
+
+              from sqlalchemy.orm import sessionmaker
+
+              Session = sessionmaker(bind=engine)  # bound session
+              session = Session()
+
+              bidding_area1 = BiddingArea("NO1", "Elspot NO1")
+              session.add(bidding_area1)
+
+              session.add_all(
+              [
+              BiddingArea("NO2", "Elspot NO2"),
+              BiddingArea("NO3", "Elspot NO3"),
+              BiddingArea("NO4", "Elspot NO4"),
+              BiddingArea("NO5", "Elspot NO5"),
+              ]
+              )
+
+              production_type_B37 = ProductionType("B37", "Thermal unspecified")
+              production_type_B30 = ProductionType("B30", "Wind unspecified")
+
+              session.add_all(
+              [
+              ProductionType("B19", "Wind Onshore"),
+              ProductionType("B10", "Hydro-electric pure pumped storage head installation"),
+              ProductionType("B11", "Hydro Run-of-river head installation"),
+              ProductionType("B12", "Hydro-electric storage head installation"),
+              ProductionType("A04", "Generation"),
+              production_type_B37,
+              production_type_B30,
+              ]
+              )
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Creating instances

    + 
    +            
+              # adding some production plans...
+
+              session.add(
+              ProductionPlan(
+              datetime.datetime.now(),
+              datetime.datetime(2022, 11, 2, 1, 0),
+              production_type_B37,
+              bidding_area1,
+              decimal.Decimal("80.5"),
+              )
+              )
+
+              production_plan2 = ProductionPlan(
+              datetime.datetime.now(),
+              datetime.datetime(2022, 11, 2, 2, 0),
+              production_type_B37,
+              bidding_area1,
+              decimal.Decimal("90.5"),
+              )
+
+              session.add(production_plan2)
+
+              session.flush()  # execute pending operations
+              session.commit()  # execute and commit pending operations
+
+              production_plan2.value = decimal.Decimal("70.5")
+              production_plan2 in session
+              # Out: True
+
+              session.commit()
+            
+
    +
    + +
    +

    Example (cont...)

    +

    Queries

    + 
    +            
+              import pandas as pd
+
+              session.query(ProductionPlan).order_by(ProductionPlan.start_time)  # returns a Query instance
+              session.query(ProductionPlan).order_by(ProductionPlan.start_time).all()  # returns an object-list
+
+              # return all production plans where start_time after 2022-09-01 00:00
+              session.query(ProductionPlan).filter(ProductionPlan.start_time > datetime.datetime(2022, 9, 1, 0, 0)).all()
+
+              # return production plans with value > 80
+              query = session.query(ProductionPlan).filter(ProductionPlan.value > 80).order_by(ProductionPlan.start_time)
+              query.count()  # returns 1
+              production_plan = query.first()  # returns the first object (element)
+              production_plan = query.one()  # raises NoResultFound exception or MultipleResultsFound in case elements != 1
+
+              # generate Pandas DataFrame from a query or entire table
+              df = pd.read_sql_table("my_table", con=session.get_bind())  # or con=engine
+              df = pd.read_sql_query(query.statement, engine)
+
+              # return production plans with production type 'B37'
+              session.query(ProductionPlan).filter(
+              ProductionPlan.production_type_id == ProductionType.production_type_id
+              ).filter(ProductionType.code == "B37").all()
+              session.query(ProductionPlan).join(ProductionType).filter(ProductionType.code == "B37").all()
+              session.query(ProductionPlan).filter(ProductionPlan.production_type == production_type_B37).all()
+              session.query(
+              ProductionPlan
+              ).from_statement(
+              text(
+              "SELECT pp.* FROM production_plans pp, production_types pt "
+              "WHERE pp.production_type_id = pt.production_type_id AND pt.code=:code"
+              )
+              ).params(code="B37").all()
+            
+
    +
    + +
    +

    Q & A

    +
    + + + + + + + + + + + + diff --git a/reveal.js/template.html b/reveal.js/template.html new file mode 100644 index 0000000..51ba440 --- /dev/null +++ b/reveal.js/template.html @@ -0,0 +1,55 @@ + + + + + Title + + + + + + + + + + + + + + + + +
    + +
    + +
    +
    + + + + + + + + + + diff --git a/reveal.js/template_statnett.html b/reveal.js/template_statnett.html new file mode 100644 index 0000000..e47b280 --- /dev/null +++ b/reveal.js/template_statnett.html @@ -0,0 +1,56 @@ + + + + + Title + + + + + + + + + + + + + + + + + +
    + +
    + +
    +
    + + + + + + + + + + diff --git a/reveal.js/timetravel.html b/reveal.js/timetravel.html deleted file mode 100644 index b297e2f..0000000 --- a/reveal.js/timetravel.html +++ /dev/null @@ -1,127 +0,0 @@ - - - - - Solving problems using time travel - - - - - - - - - - - - - - - -
    - - -
    - -
    -

    Solving problems using time travel

    -

    Beryl @ Fifty

    -
    -

    Tomas Robertson (team ACE-OL) & Simeon Simeonov (team Forecasts)

    -
    - - -
    -

    What is time travel???

    -
    -

    Time travel - our ability to look at our input data at the state it was at a specific point in time (not only at its last state).

    -

    Time travel is achieved by adding the record_created_time column to our DB tables and storing Kafka's created_time value (converted to UTC).

    -
    -

    The main concepts around how and why were presented by Peter Sandberg.

    -
    - -
    -

    Solving problems

    -
    -

    The following spike was observed 2021-10-06 around 10:15 CET @ NO4

    - -
    - -
    -

    Solving problems (cont)

    -
    -

    No spikes shown in Grafana

    - -
    - -
    -

    Solving problems with time travel

    -
    - 
    -                        
-                            SELECT record_created_time, start_time, value
-                            FROM misc.app_odin_ace_ol_ba_10s_avro_v01
-                            WHERE bidding_area_name = 'NO4' AND start_time = '2021-10-06 08:14:10'
-                            ORDER BY record_created_time;
-                        
-
    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    record_created_timestart_timevalue
    2021-10-06 08:15:52.2582021-10-06 08:14:10-370.37683609008127
    2021-10-06 08:19:06.2222021-10-06 08:14:1025.183745117193457
    2021-10-06 08:21:56.2072021-10-06 08:14:1034.78731922151518
    -
    - -
    -

    Q & A

    -
    - -
    -
    - - - - - - - - - - - -- cgit v1.3