summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSimeon Simeonov2022-01-26 18:46:50 +0100
committerSimeon Simeonov2022-01-26 18:46:50 +0100
commit29c52d98bf7feb132a1857cce2058ec7134e4b0f (patch)
tree632d989dd16a2cb92820697af49b89f4fca1f9fb
parent85c1bd41d6b3b985bb6794ebe066128c51e7c8fd (diff)
Add postgresql_tuning.html python.html timetravel.html and update sqlalchemy.html and demo.html
Diffstat
-rw-r--r--reveal.js/demo.html58
-rw-r--r--reveal.js/postgresql_tuning.html152
-rw-r--r--reveal.js/python.html432
-rwxr-xr-xreveal.js/sqlalchemy.html5
-rw-r--r--reveal.js/timetravel.html127
5 files changed, 739 insertions, 35 deletions
diff --git a/reveal.js/demo.html b/reveal.js/demo.html
index 71cac98..ea42adf 100644
--- a/reveal.js/demo.html
+++ b/reveal.js/demo.html
@@ -19,7 +19,7 @@
19 <link rel="stylesheet" href="dist/theme/black.css" id="theme"> 19 <link rel="stylesheet" href="dist/theme/black.css" id="theme">
20 20
21 <!-- Theme used for syntax highlighting of code --> 21 <!-- Theme used for syntax highlighting of code -->
22 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme"> 22 <link rel="stylesheet" href="plugin/highlight/monokai.css">
23 </head> 23 </head>
24 24
25 <body> 25 <body>
@@ -186,7 +186,7 @@
186 Write content using inline or external Markdown. 186 Write content using inline or external Markdown.
187 Instructions and more info available in the [docs](https://revealjs.com/markdown/). 187 Instructions and more info available in the [docs](https://revealjs.com/markdown/).
188 188
189 ```[] 189 ```html []
190 <section data-markdown> 190 <section data-markdown>
191 ## Markdown support 191 ## Markdown support
192 192
@@ -249,17 +249,17 @@
249 <p> 249 <p>
250 reveal.js comes with a few themes built in: <br> 250 reveal.js comes with a few themes built in: <br>
251 <!-- Hacks to swap themes after the page has loaded. Not flexible and only intended for the reveal.js demo deck. --> 251 <!-- Hacks to swap themes after the page has loaded. Not flexible and only intended for the reveal.js demo deck. -->
252 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/black.css'); return false;">Black (default)</a> - 252 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/black.css'); return false;">Black (default)</a> -
253 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/white.css'); return false;">White</a> - 253 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/white.css'); return false;">White</a> -
254 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/league.css'); return false;">League</a> - 254 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/league.css'); return false;">League</a> -
255 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/sky.css'); return false;">Sky</a> - 255 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/sky.css'); return false;">Sky</a> -
256 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/beige.css'); return false;">Beige</a> - 256 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/beige.css'); return false;">Beige</a> -
257 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/simple.css'); return false;">Simple</a> <br> 257 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/simple.css'); return false;">Simple</a> <br>
258 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/serif.css'); return false;">Serif</a> - 258 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/serif.css'); return false;">Serif</a> -
259 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/blood.css'); return false;">Blood</a> - 259 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/blood.css'); return false;">Blood</a> -
260 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/night.css'); return false;">Night</a> - 260 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/night.css'); return false;">Night</a> -
261 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/moon.css'); return false;">Moon</a> - 261 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/moon.css'); return false;">Moon</a> -
262 <a href="#" onclick="document.getElementById('theme').setAttribute('href','css/theme/solarized.css'); return false;">Solarized</a> 262 <a href="#" onclick="document.getElementById('theme').setAttribute('href','dist/theme/solarized.css'); return false;">Solarized</a>
263 </p> 263 </p>
264 </section> 264 </section>
265 265
@@ -281,7 +281,7 @@
281 <h2>Tiled Backgrounds</h2> 281 <h2>Tiled Backgrounds</h2>
282 <pre><code class="hljs html" style="word-wrap: break-word;">&lt;section data-background="image.png" data-background-repeat="repeat" data-background-size="100px"&gt;</code></pre> 282 <pre><code class="hljs html" style="word-wrap: break-word;">&lt;section data-background="image.png" data-background-repeat="repeat" data-background-size="100px"&gt;</code></pre>
283 </section> 283 </section>
284 <section data-background-video="https://s3.amazonaws.com/static.slid.es/site/homepage/v1/homepage-video-editor.mp4" data-background-color="#000000"> 284 <section data-background-video="https://static.slid.es/site/homepage/v1/homepage-video-editor.mp4" data-background-color="#000000">
285 <div style="background-color: rgba(0, 0, 0, 0.9); color: #fff; padding: 20px;"> 285 <div style="background-color: rgba(0, 0, 0, 0.9); color: #fff; padding: 20px;">
286 <h2>Video Backgrounds</h2> 286 <h2>Video Backgrounds</h2>
287 <pre><code class="hljs html" style="word-wrap: break-word;">&lt;section data-background-video="video.mp4,video.webm"&gt;</code></pre> 287 <pre><code class="hljs html" style="word-wrap: break-word;">&lt;section data-background-video="video.mp4,video.webm"&gt;</code></pre>
@@ -451,25 +451,23 @@ Reveal.on( 'customevent', function() {
451 </div> 451 </div>
452 452
453 <script src="dist/reveal.js"></script> 453 <script src="dist/reveal.js"></script>
454 <script src="plugin/zoom/zoom.js"></script>
455 <script src="plugin/notes/notes.js"></script>
456 <script src="plugin/search/search.js"></script>
457 <script src="plugin/markdown/markdown.js"></script>
458 <script src="plugin/highlight/highlight.js"></script>
459 <script> 454 <script>
455 Reveal.initialize();
460 456
461 // Also available as an ES module, see: 457 const loadScript = src => {
462 // https://revealjs.com/initialization/ 458 return new Promise((resolve, reject) => {
463 Reveal.initialize({ 459 const script = document.createElement('script');
464 controls: true, 460 script.type = 'text/javascript';
465 progress: true, 461 script.onload = resolve;
466 center: true, 462 script.onerror = reject;
467 hash: true, 463 script.src = src;
468 464 document.head.append(script);
469 // Learn about plugins: https://revealjs.com/plugins/ 465 })
470 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ] 466 }
471 });
472 467
468 loadScript( 'plugin/highlight/highlight.js' ).then(() => {
469 Reveal.registerPlugin( RevealHighlight )
470 })
473 </script> 471 </script>
474 472
475 </body> 473 </body>
diff --git a/reveal.js/postgresql_tuning.html b/reveal.js/postgresql_tuning.html
new file mode 100644
index 0000000..d7e3466
--- /dev/null
+++ b/reveal.js/postgresql_tuning.html
@@ -0,0 +1,152 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>SQLAlchemy</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>PostgreSQL tuning</h2>
28 <h4>Data Science @ Beryl</h4>
29 </br>
30 <p><small>Simeon Simeonov</small></p>
31 </section>
32
33 <section>
34
35 <section id="fragments">
36 <h2>Agenda</h2>
37 </br>
38 <ul>
39 <span class="fragment"><li>Generic tools for gathering information</li></span>
40 <span class="fragment"><li>Memory settings</li></span>
41 <span class="fragment"><li>Logging and performance reports</li></span>
42 <span class="fragment"><li>Other tools for analysis</li></span>
43 </ul>
44 </section>
45
46 </section>
47
48 <section>
49 <h2>General tools for gathering information</h2>
50 </br>
51 <pre data-id="code-animation">
52 <code class="bash" data-trim type="text/template">
53 # shell
54 # fetch information from the OS
55 cat /proc/cpuinfo
56 cat /proc/meminfo
57 sysctl -a | grep shm # get kernel parameters of interest
58 </code>
59 <code class="sql" data-trim type="text/template">
60 -- SQL
61 -- show the current values of all settings
62 SHOW ALL;
63
64 -- display even more than all...
65 SELECT * FROM pg_settings;
66
67 -- opening postgresql.conf and reading the comments - the old school approach
68 </code>
69 <code class="bash" data-trim type="text/template">
70 # old school: edit postgresql.conf and read the comments
71 </code>
72 </pre>
73 </section>
74
75 <section>
76 <h2>Memory settings</h2>
77 </br>
78 <ul>
79 <li><em>shared_buffers</em> - how much memory is dedicated to PostgreSQL to use for caching data - for a system with 1GB or more of RAM, a reasonable starting value for <em>shared_buffers</em> is 25% of the system memory (128MB -> 1GB)</li>
80 <li><em>effective_cache_size</em> - how much memory we expect to be available in the OS and PostgreSQL buffer caches, not an allocation - used only by the PostgreSQL query planner to figure out whether plans it's considering would be expected to fit in RAM or not - 1/2 of total memory would be a normal conservative setting (4GB -> 8GB)</li>
81 <li><em>work_mem</em> - the base maximum amount of memory to be used by a query operation (such as a sort or hash table) before writing to temporary disk files - for a complex query, several sort or hash operations might be running in parallel; each operation will generally be allowed to use as much memory as this value specifies (4MB -> 8MB)</li>
82 <li><em>maintenance_work_mem</em> - the maximum amount of memory to be used by maintenance operations, such as <em>VACUUM</em> and <em>CREATE INDEX</em>. It's safe to set this value significantly larger than <em>work_mem</em> (64MB -> 256MB)</li>
83 </ul>
84 </section>
85
86 <section>
87 <h2>Logging and performance reports</h2>
88 </br>
89 <p><em>pgBadger - a fast PostgreSQL log analysis report</em> can be used for general analysis.</p>
90 <ul>
91 <li><em>log_checkpoints</em> - checkpoints and restartpoints are logged in the server log. Some statistics are included in the log messages, including the number of buffers written and the time spent writing them</li>
92 <li><em>log_connections</em> - each attempted connection to the server to be logged, as well as successful completion of client authentication</li>
93 <li><em>log_disconnections</em> - provides information similar to <em>log_connections</em>, plus the duration of the session</li>
94 <li><em>log_line_prefix</em> - set to '%t [%p]: user=%u,db=%d,app=%a,client=%h '</li>
95 <li><em>log_lock_waits</em> - log message is produced when a session waits longer than deadlock_timeout to acquire a lock. This is useful in determining if lock waits are causing poor performance</li>
96 <li><em>log_temp_files</em> - when set to <em>0</em>, a log entry is emitted for each temporary file when it is deleted</li>
97 <li><em>log_autovacuum_min_duration</em> - set to <em>0</em> it logs all autovacuum actions</li>
98 </ul>
99 </section>
100
101 <section>
102 <h2>Other tools for analysis</h2>
103 </br>
104 <ul>
105 <li><em>ANALYZE</em> - collects statistics about the contents of tables in the database, and stores the results in the pg_statistic system catalog. Subsequently, the query planner uses these statistics to help determine the most efficient execution plans for queries.</li>
106 <li><em>VACUUM</em> - reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. (<em>VACUUM</em> vs. <em>VACUUM FULL</em>)</li>
107 </ul>
108 </section>
109
110 <section>
111 <h2>Sources</h2>
112 </br>
113 <p>
114 <a href="https://www.postgresql.org/docs/">https://www.postgresql.org/docs/</a> - The official documentation
115 </p>
116 <p>
117 <a href="https://wiki.postgresql.org">https://wiki.postgresql.org</a> - The official Wiki
118 </p>
119 </section>
120
121 <section>
122 <h1>Q &amp; A</h1>
123 </section>
124
125 </div>
126 </div>
127
128 <script src="dist/reveal.js"></script>
129 <script src="plugin/zoom/zoom.js"></script>
130 <script src="plugin/notes/notes.js"></script>
131 <script src="plugin/search/search.js"></script>
132 <script src="plugin/markdown/markdown.js"></script>
133 <script src="plugin/highlight/highlight.js"></script>
134 <script>
135
136 // Also available as an ES module, see:
137 // https://revealjs.netlify.app/initialization/
138 Reveal.initialize({
139 controls: true,
140 progress: true,
141 center: true,
142 hash: true,
143
144 // Learn about plugins: https://revealjs.netlify.app/plugins/
145 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
146 });
147 Reveal.configure({ pdfSeparateFragments: false });
148
149 </script>
150
151 </body>
152</html>
diff --git a/reveal.js/python.html b/reveal.js/python.html
new file mode 100644
index 0000000..5190dfb
--- /dev/null
+++ b/reveal.js/python.html
@@ -0,0 +1,432 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>Introduction to Python</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/statnett.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>Introduction to Python</h2>
28 <h4>Data Engineering @ Statnett</h4>
29 </br>
30 <p><small>Simeon Simeonov</small></p>
31 </section>
32
33 <section>
34 <h2>Goals</h2>
35 </br>
36 <ul>
37 <li><p>Present the Python programming language in a different way than <a href="https://docs.python.org">https://docs.python.org</a></p></li>
38 <li><p>Avoid information overload</p></li>
39 <li><p>Use examples and interaction rather than documents and slides</p></li>
40 </ul>
41 </section>
42
43 <section>
44 <h2>Preliminary plan</h2>
45 </br>
46 <ul>
47 <li><p>Basics: About the language, the Python eco-system, types, scopes, modules and functions</p></li>
48 <li><p>Control flow: if / for / while / try, iterators, decorators, "tactical programming" tips</p></li>
49 <li><p>Object orientation: How Python "really works"</p></li>
50 <li><p>A brief tour through Python's standard library</p></li>
51 <li><p>Code design and best practices: How to design your code</p></li>
52 </ul>
53 </section>
54
55 <section>
56 <h2>What is Python?</h2>
57 </br>
58 <ul>
59 <li><p>Python is an interpreted high-level general-purpose programming language - advanced through the Python Enhancement Proposal (PEP) process</p></li>
60 <li><p>CPython is the reference implementation of Python, written in C (alternatives: pypy, jython)</p></li>
61 <li><p>python - interpreter and interpreter shell (alternatives: ipython, bpython)</p></li>
62 <li><p>libpython</p></li>
63 <li><p>Calling C from Python: Cython, CFFI, ctypes</p></li>
64 </ul>
65 </section>
66
67 <section>
68 <h2>Philosophy</h2>
69 </br>
70 <pre data-id="code-animation">
71 <code class="bash" data-trim type="text/template">
72 $ python
73 </code>
74 </pre>
75 <pre data-id="code-animation">
76 <code class="python" data-trim type="text/template">
77 import this
78
79 # The Zen of Python, by Tim Peters
80
81 # Beautiful is better than ugly.
82 # Explicit is better than implicit.
83 # Simple is better than complex.
84 # Complex is better than complicated.
85 # Flat is better than nested.
86 # Sparse is better than dense.
87 # Readability counts.
88 # Special cases aren't special enough to break the rules.
89 # Although practicality beats purity.
90 # Errors should never pass silently.
91 # Unless explicitly silenced.
92 # In the face of ambiguity, refuse the temptation to guess.
93 # There should be one-- and preferably only one --obvious way to do it.
94 # Although that way may not be obvious at first unless you're Dutch.
95 # Now is better than never.
96 # Although never is often better than *right* now.
97 # If the implementation is hard to explain, it's a bad idea.
98 # If the implementation is easy to explain, it may be a good idea.
99 # Namespaces are one honking great idea -- let's do more of those!
100 </code>
101 </pre>
102 </section>
103
104 <section>
105 <h2>Built-in functions</h2>
106 </br>
107 <p>Few built-in functions.</p>
108 <p><a href="https://docs.python.org/3/library/functions.html">https://docs.python.org/3/library/functions.html</a></p>
109 <ul>
110 <li><em>dir([obj])</em> - returns a list of valid attributes for that object</li>
111 <li><em>id(obj)</em> - returns the "identity" of an object - an integer which is guaranteed to be unique</li>
112 <li><em>print(...)</em> - prints objects to a text stream</li>
113 <li><em>str(...)</em> - returns a string version of object</li>
114 <li><em>type(obj)</em> - returns the type of an object</li>
115 </ul>
116 </section>
117
118 <section>
119 <h2>Common built-in types</h2>
120 </br>
121 <p>Python uses <a href="https://en.wikipedia.org/wiki/Duck_typing"><em>duck typing</em></a> and has typed objects but untyped variable names.</p>
122 <p>Type constraints are not checked at compile time; rather, operations on an object may fail, signifying that the given object is not of a suitable type. Despite being dynamically-typed, Python is strongly-typed, forbidding operations that are not well-defined (for example, adding a number to a string) rather than silently attempting to make sense of them.</p>
123 <pre data-id="code-animation">
124 <code class="python" data-trim type="text/template">
125 s = 'foo' # this is a string / str, same as str('foo'), may be encoded, immutable
126 b = b'foo' # bytes, same as bytes('foo', 'utf-8'), may be decoded, immutable
127 i = 6 # int, same as int('6'), immutable
128 f = 0.1 # float, same as float('0.1'), immutable
129 b = False # bool, same as bool(0), bool(''), bool(None)... immutable / constant
130 n = None # NoneType, similar to 'null' in other languages, immutable / constant
131 l = [1, False, 'foo'] # list, same as list((1, False, 'foo'))
132 t = (1, False, 'foo') # tuple, same as tuple([1, False, 'foo']), immutable
133 d = {'foo': 1, 'bar': 8} # dict, same as dict(foo=1, bar=8), similar to hash in other languages
134 s = {'foo', 'bar', 1, 1, 4} # set, same as set(['foo', 'bar', 1, 1, 4]), removes duplicates
135 </code>
136 </pre>
137 <p>Classes, functions, modules, .... and even types are simply other types :)</p>
138 </section>
139
140 <section>
141 <h2>Modules</h2>
142 </br>
143 <p>A module is a file containing Python definitions and statements. The file name is the module name with the suffix <em>.py</em> appended. Within a module, the module's name (as a string) is available as the value of the global variable <em>__name__</em></p>
144 <p>When a module named <em>foo</em> is imported, the interpreter first searches for a built-in module with that name (<em>sys.builtin_module_names</em>). If not found, it then searches for a file named <em>foo.py</em> in a list of directories given by the variable <em>sys.path</em>. <em>sys.path</em> is initialized from these locations:</p>
145 <ul>
146 <li>the directory containing the input script (or the current directory when no file is specified)</li>
147 <li><em>PYTHONPATH</em> - env. variable - a list of directory names</li>
148 <li>the installation-dependent default locations</li>
149 </ul>
150
151 <p>The module is then imported only once and "cached" in <em>sys.modules</em></p>
152 </section>
153
154 <section>
155 <h2>Packages</h2>
156 </br>
157 <p><a href="https://docs.python.org/3/tutorial/modules.html#packages">Packages</a> are a way of structuring Python's module namespace by using "dotted module names"</p>
158 <p>The import statement combines two operations:</p>
159 <ul>
160 <li>it searches for the named module</li>
161 <li>it binds the results of that search to a name in the <em>local scope</em></li>
162 </ul>
163 <pre data-id="code-animation">
164 <code class="python" data-trim type="text/template">
165 # bar.py then bar/__init__.py will be considered, the first match executed and bound to 'bar'
166 import bar
167
168 import mymodule.foo # implicitly executes mymodule.py, mymodule/__init__.py and mymodule/foo/__init__.py
169
170 import numpy as np # will be bound as 'np' instead of 'numpy'. N.B. __name__ is still 'numpy'
171
172 import some.extremely.deep.path.Animal as Animal # "sacrifice" the namespace in the name of convinience
173
174 from sys import path # execute sys and only import the 'path' attribute into local scope as 'path'
175
176 # relative imports must be explicit in Python 3
177 from .othermodule import something # expects that current module and 'othermodule' are in the same
178 # package (containing __init__.py)
179
180 from sys import * # NO! Bad programming practice since 1879
181 </code>
182 </pre>
183 </section>
184
185 <section>
186 <h2>Creating and maintaining a Python environment</h2>
187 </br>
188 <p>Python's official package repository is <a href="https://pypi.org"><em>PyPi (https://pypi.org)</em></a>, while Python's official package installer is <a href="https://pypi.org/project/pip/"><em>pip (https://pypi.org/project/pip/)</em></a></p>
189 <p>A Python environment is the physical and logical arrangement of Python modules and packages. Several options exist:</p>
190 <ul>
191 <li>using a proper operating system :) (symlinks, real commercial support etc.)</li>
192 <li>using <em>venv</em></li>
193 <li>using higher level tools like <em>poetry</em></li>
194 <li>using a mixture / cocktail of all of the above :)</li>
195 </ul>
196 </section>
197
198 <section>
199 <h2>Creating and maintaining a Python environment (cont...)</h2>
200 </br>
201 <p>Desired qualities for a flexible Python environment:</p>
202 <ul>
203 <li>easy to create and (un)load</li>
204 <li>do not require extra privileges</li>
205 <li>don't repeat yourself (DRY)</li>
206 <li>easy to update without breaking the API</li>
207 <li>easy to debug</li>
208 <li>play nicely with the VCS (git)</li>
209 </ul>
210 </section>
211
212 <section>
213 <h3>Creating and maintaining a Python environment (cont...)</h3>
214 </br>
215 <p>Exploting the operating system can be done by:</p>
216 <ul>
217 <li>(re)defining <em>PYTHONPATH</em></li>
218 <li>using symlinks to point at packages placed at different locations</li>
219 </section>
220
221 <section>
222 <h3>Creating and maintaining a Python environment (cont...)</h3>
223 </br>
224 <p>Using <em>venv</em> can be done by directly invoking <em>python</em>:</p>
225 <pre data-id="code-animation">
226 <code class="bash" data-trim type="text/template">
227 # create a virtual environment
228 python -m venv my_virtual_env
229 python -m venv --system-site-packages my_virtual_env
230
231 # load, use and unload the virtual environment
232 source my_virtual_env/bin/activate
233 pip install sqlalchemy
234 deactivate
235
236 # one can alternatively use the python "wrapper" of the virtual env
237 my_virtual_env/bin/python -m pip install sqlalchemy
238 </code>
239 </pre>
240 <p><em>--system-site-packages</em> will keep the original <em>site-packages</em> folders at the end of <em>sys.path</em></p>
241 </section>
242
243 <section>
244 <h3>Creating and maintaining a Python environment (cont...)</h3>
245 </br>
246 <p><em>Poetry</em> (<a href="https://python-poetry.org"><em>https://python-poetry.org</em></a>) is the prefered environment and dependency management tool at Statnett.</p>
247 <pre data-id="code-animation">
248 <code class="bash" data-trim type="text/template">
249 # create project and a virtual environment from scratch
250 poetry new my-project
251
252 # ... or use Poetry with an existing one
253 cd my-project
254 poetry init
255
256 # edit pyproject.toml for your needs (f.i. add dependencies, metadata ... etc),
257 # create virtual environment and install dependencies
258 poetry install
259 # finally commit your poetry.lock file to version control
260
261 # update all dependencies
262 poetry update
263 </code>
264 </pre>
265 <p>For more info: <a href="https://python-poetry.org/docs/basic-usage/"><em>https://python-poetry.org/docs/basic-usage/</em></a></p>
266 </section>
267
268 <section>
269 <h2>Mutables vs. immutables</h2>
270 <p>Immutable object is an object with a fixed value. Immutable objects include <em>bool, int, float, str, bytes</em> and <em>tuples</em>. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key in a dictionary.</p>
271 <p>All objects that are not immutable are... mutable. All <em>hashable objects</em> <b>should</b> be immutable or use <em>id()</em>.</p>
272 <pre data-id="code-animation">
273 <code class="python" data-trim type="text/template">
274 i = 1
275 id(i) # returns f.i. 9788992
276 i += 1 # same as i = i + 1
277 id(i) # returns a different value, hence - a brand new object
278
279 s = 'Hello'
280 s += ' World' # s is now a different object
281 s[0] # 'H'
282 s[0] = 'h' # TypeError: 'str' object does not support item assignment
283
284 t = (1, 4) # tuple
285 l = [1, 4] # list
286 hash(t) # returns f.i. -6333845781340707986
287 hash(l) # TypeError: unhashable type: 'list'
288
289 s = 'the long and winding road'
290 s2 = 'the long and winding road'
291
292 # check if s and s2 are the same object:
293 id(s) # Out: 139858905258704
294 id(s2) # Out: 139858926037472
295
296 # the hash should be the same
297 hash(s) # Out: 7030216208569256362
298 hash(s2) # Out: 7030216208569256362
299 </code>
300 </pre>
301 </section>
302
303 <section>
304 <h2>Functions</h2>
305 </br>
306 <p>A function is a sequence of program instructions that performs a specific task, packaged as a unit.</p>
307 <p>Functions let you:</p>
308 <ul>
309 <li>reuse code across several programs / projects</li>
310 <li>minimize code duplication</li>
311 <li>devide larger programming tasks</li>
312 <li>hide implementation details</li>
313 <li>improve readability</li>
314 <li>improve traceability</li>
315 </ul>
316 <p>Function calls bring some overhead pushing / popping function-data into / from stack.</p>
317 <p>Important definitions (may have different meanings in different programming languages):</p>
318 <ul>
319 <li><em>parameter / formal parameter</em> - variable / data provided as input to the function</li>
320 <li><em>argument / actual parameter</em> - local variable / data to the given function</li>
321 </ul>
322 <p>The keyword <em>def</em> introduces a function definition.</p>
323 </section>
324
325 <section>
326 <h2>Functions (cont...)</h2>
327 </br>
328 <pre data-id="code-animation">
329 <code class="python" data-trim type="text/template">
330 def add(a, b): # - function definition / header
331 """Function for adding integers""" # - docstring
332 result = a + b
333 a = 5
334 return result # - function that does not contain return, implicitly returns None
335
336 a_param = 9
337 b_param = -2
338 add(a_param, b_param) # Out: 7
339
340 # integers are immutable and a_param will remain unchanged
341 print(a_param) # Out: 9
342
343
344 def addl(a, b):
345 """Function for adding two lists"""
346 result = a + b
347 a += [5]
348 return result
349
350 a_param = [9]
351 b_param = [-2]
352 addl(a_param, b_param) # Out: [9, -2]
353 # lists are mutable and a_param will be changed
354 print(a_param) # Out: [5]
355 </code>
356 </pre>
357 </section>
358
359 <section>
360 <h2>Functions</h2>
361 </br>
362 <h3>To be continued in the next episode....</h3>
363 </section>
364
365 <section>
366 <h2>Scopes in Python</h2>
367 </br>
368 <ul>
369 <li><em>local</em> - assigned names are local unless declared <em>global</em></li>
370 <li><em>enclosed</em> - the scope of the variable inside a function with a nested function</li>
371 <li><em>global - global for the current module</em></li>
372 <li><em>built-in</em></li>
373 </ul>
374 <p><em>locals()</em> and <em>globals()</em> return dicts of symbols for their respective scopes</p>
375 <pre data-id="code-animation">
376 <code class="python" data-trim type="text/template">
377 num1, num2 = 7, 8 # module globals
378
379 def print_numbers():
380 print(num1, num2) # OK, these are module globals
381 num3 = 100
382 print(num3) # prints 100, num3 is in the function (local) scope
383 global num4 # assignes / references num4 to / in the global scope
384 num4 = 99
385 id = 200 # new symbol in local scope
386 # id(num4) # will not yield the expected result (raises TypeError)
387
388 def print_numbers2():
389 print(num3) # OK, enclosed scope
390
391 print_numbers2() # prints 100
392
393 print_numbers()
394 # print(num3) # Raises NameError - why?
395 print(num4) # Prints 99 - why?
396 # print_numbers2() # Raises NameError
397 </code>
398 </pre>
399 </section>
400
401 <section>
402 <h1>Q &amp; A</h1>
403 </section>
404
405 </div>
406 </div>
407
408 <script src="dist/reveal.js"></script>
409 <script src="plugin/zoom/zoom.js"></script>
410 <script src="plugin/notes/notes.js"></script>
411 <script src="plugin/search/search.js"></script>
412 <script src="plugin/markdown/markdown.js"></script>
413 <script src="plugin/highlight/highlight.js"></script>
414 <script>
415
416 // Also available as an ES module, see:
417 // https://revealjs.netlify.app/initialization/
418 Reveal.initialize({
419 controls: true,
420 progress: true,
421 center: true,
422 hash: true,
423
424 // Learn about plugins: https://revealjs.netlify.app/plugins/
425 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
426 });
427 Reveal.configure({ pdfSeparateFragments: false });
428
429 </script>
430
431 </body>
432</html>
diff --git a/reveal.js/sqlalchemy.html b/reveal.js/sqlalchemy.html
index 6acd9b7..96f83e7 100755
--- a/reveal.js/sqlalchemy.html
+++ b/reveal.js/sqlalchemy.html
@@ -62,7 +62,6 @@
62 <li><p>object-orientation - you work with objects instead of tables and rows</p></li> 62 <li><p>object-orientation - you work with objects instead of tables and rows</p></li>
63 <li><p>performance - exploits the likehood of reusing a particular query</p></li> 63 <li><p>performance - exploits the likehood of reusing a particular query</p></li>
64 <li><p>flexibility - you can override almost anything</p></li> 64 <li><p>flexibility - you can override almost anything</p></li>
65 <li><p>0.0125 karma points for each MB transferred (2021)</p></li>
66 </ul> 65 </ul>
67 </section> 66 </section>
68 67
@@ -302,10 +301,6 @@
302 </pre> 301 </pre>
303 </section> 302 </section>
304 303
305 <section data-background="http://i.giphy.com/90F8aUepslB84.gif">
306 <h2>... there is more???</h2>
307 </section>
308
309 <section> 304 <section>
310 <h2>Some nice features</h2> 305 <h2>Some nice features</h2>
311 <pre data-id="code-animation"> 306 <pre data-id="code-animation">
diff --git a/reveal.js/timetravel.html b/reveal.js/timetravel.html
new file mode 100644
index 0000000..e073f92
--- /dev/null
+++ b/reveal.js/timetravel.html
@@ -0,0 +1,127 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8">
5 <title>Solving problems using time travel</title>
6 <meta name="author" content="Simeon Simeonov">
7 <meta name="apple-mobile-web-app-capable" content="yes">
8 <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <link rel="stylesheet" href="dist/reset.css">
12 <link rel="stylesheet" href="dist/reveal.css">
13
14 <link rel="stylesheet" href="dist/theme/fifty.css" id="theme">
15
16 <!-- Theme used for syntax highlighting of code -->
17 <link rel="stylesheet" href="plugin/highlight/monokai.css" id="highlight-theme">
18 <!-- <link rel="stylesheet" href="plugin/highlight/zenburn.css" id="highlight-theme"> -->
19 </head>
20 <body>
21 <div class="reveal">
22
23 <!-- Any section element inside of this container is displayed as a slide -->
24 <div class="slides">
25
26 <section>
27 <h2>Solving problems using <em>time travel</em></h2>
28 <h4>Beryl @ Fifty</h4>
29 </br>
30 <p><small>Tomas Robertson (team ACE-OL) &amp; Simeon Simeonov (team Forecasts)</small></p>
31 </section>
32
33
34 <section>
35 <h2>What is <em>time travel</em>???</h2>
36 <br/>
37 <p><em>Time travel</em> - our ability to look at our input data at the state it was at a specific point in time (not only at its last state).</p>
38 <p><em>Time travel</em> is achieved by adding the <em>record_created_time</em> column to our DB tables and storing Kafka's <em>created_time</em> value (converted to UTC).</p>
39 <br/>
40 <p>The main concepts around how and why were presented by Peter Sandberg.</p>
41 </section>
42
43 <section>
44 <h2>Solving problems</h2>
45 <br/>
46 <p>The following spike was observed 2021-10-06 around 10:15 CET @ NO4</p>
47 <img src="images/timetravel/ace_ol_spike.png"></img>
48 </section>
49
50 <section>
51 <h2>Solving problems (cont)</h2>
52 <br/>
53 <p>No spikes shown in Grafana</p>
54 <img src="images/timetravel/ace_ol_spike_grafana.png"></img>
55 </section>
56
57 <section>
58 <h2>Solving problems with <em>time travel</em></h2>
59 <br/>
60 <pre data-id="code-animation">
61 <code class="sql" data-trim type="text/template">
62 SELECT record_created_time, start_time, value
63 FROM misc.app_odin_ace_ol_ba_10s_avro_v01
64 WHERE bidding_area_name = 'NO4' AND start_time = '2021-10-06 08:14:10'
65 ORDER BY record_created_time;
66 </code>
67 </pre>
68
69 <table>
70 <thead>
71 <tr>
72 <th>record_created_time</th>
73 <th>start_time</th>
74 <th>value</th>
75 </tr>
76 </thead>
77 <tbody>
78 <tr>
79 <td>2021-10-06 08:15:52.258</td>
80 <td>2021-10-06 08:14:10</td>
81 <td>-370.37683609008127</td>
82 </tr>
83 <tr>
84 <td>2021-10-06 08:19:06.222</td>
85 <td>2021-10-06 08:14:10</td>
86 <td>25.183745117193457</td>
87 </tr>
88 <tr>
89 <td>2021-10-06 08:21:56.207</td>
90 <td>2021-10-06 08:14:10</td>
91 <td>34.78731922151518</td>
92 </tr>
93 </tbody>
94 </table>
95 </section>
96
97 <section>
98 <h1>Q &amp; A</h1>
99 </section>
100
101 </div>
102 </div>
103
104 <script src="dist/reveal.js"></script>
105 <script src="plugin/zoom/zoom.js"></script>
106 <script src="plugin/notes/notes.js"></script>
107 <script src="plugin/search/search.js"></script>
108 <script src="plugin/markdown/markdown.js"></script>
109 <script src="plugin/highlight/highlight.js"></script>
110 <script>
111
112 // Also available as an ES module, see:
113 // https://revealjs.netlify.app/initialization/
114 Reveal.initialize({
115 controls: true,
116 progress: true,
117 center: true,
118 hash: true,
119
120 // Learn about plugins: https://revealjs.netlify.app/plugins/
121 plugins: [ RevealZoom, RevealNotes, RevealSearch, RevealMarkdown, RevealHighlight ]
122 });
123
124 </script>
125
126 </body>
127</html>
generated by cgit v1.3 (git 2.53.0) at 2026-09-25 09:43:22 +0000