Coverage for .tox/coverage/lib/python3.11/site-packages/wuttjamaican/batch.py: 100%
88 statements
« prev ^ index » next coverage.py v7.3.2, created at 2025-01-09 12:13 -0600
1# -*- coding: utf-8; -*-
2################################################################################
3#
4# WuttJamaican -- Base package for Wutta Framework
5# Copyright © 2023-2024 Lance Edgar
6#
7# This file is part of Wutta Framework.
8#
9# Wutta Framework is free software: you can redistribute it and/or modify it
10# under the terms of the GNU General Public License as published by the Free
11# Software Foundation, either version 3 of the License, or (at your option) any
12# later version.
13#
14# Wutta Framework is distributed in the hope that it will be useful, but
15# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17# more details.
18#
19# You should have received a copy of the GNU General Public License along with
20# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
21#
22################################################################################
23"""
24Batch Handlers
25"""
27import datetime
28import os
29import shutil
31from wuttjamaican.app import GenericHandler
34class BatchHandler(GenericHandler):
35 """
36 Base class and *partial* default implementation for :term:`batch
37 handlers <batch handler>`.
39 This handler class "works as-is" but does not actually do
40 anything. Subclass must implement logic for various things as
41 needed, e.g.:
43 * :attr:`model_class`
44 * :meth:`init_batch()`
45 * :meth:`should_populate()`
46 * :meth:`populate()`
47 * :meth:`refresh_row()`
48 """
50 @property
51 def model_class(self):
52 """
53 Reference to the batch :term:`data model` class which this
54 batch handler is meant to work with.
56 This is expected to be a subclass of
57 :class:`~wuttjamaican.db.model.batch.BatchMixin` (among other
58 classes).
60 Subclass must define this; default is not implemented.
61 """
62 raise NotImplementedError("You must set the 'model_class' attribute "
63 f"for class '{self.__class__.__name__}'")
65 @property
66 def batch_type(self):
67 """
68 Convenience property to return the :term:`batch type` which
69 the current handler is meant to process.
71 This is effectively an alias to
72 :attr:`~wuttjamaican.db.model.batch.BatchMixin.batch_type`.
73 """
74 return self.model_class.batch_type
76 def make_batch(self, session, progress=None, **kwargs):
77 """
78 Make and return a new batch (:attr:`model_class`) instance.
80 This will create the new batch, and auto-assign its
81 :attr:`~wuttjamaican.db.model.batch.BatchMixin.id` value
82 (unless caller specifies it) by calling
83 :meth:`consume_batch_id()`.
85 It then will call :meth:`init_batch()` to perform any custom
86 initialization needed.
88 Therefore callers should use this ``make_batch()`` method, but
89 subclass should override :meth:`init_batch()` instead (if
90 needed).
92 :param session: Current :term:`db session`.
94 :param progress: Optional progress indicator factory.
96 :param \**kwargs: Additional kwargs to pass to the batch
97 constructor.
99 :returns: New batch; instance of :attr:`model_class`.
100 """
101 # generate new ID unless caller specifies
102 if 'id' not in kwargs:
103 kwargs['id'] = self.consume_batch_id(session)
105 # make batch
106 batch = self.model_class(**kwargs)
107 self.init_batch(batch, session=session, progress=progress, **kwargs)
108 return batch
110 def consume_batch_id(self, session, as_str=False):
111 """
112 Fetch a new batch ID from the counter, and return it.
114 This may be called automatically from :meth:`make_batch()`.
116 :param session: Current :term:`db session`.
118 :param as_str: Indicates the return value should be a string
119 instead of integer.
121 :returns: Batch ID as integer, or zero-padded 8-char string.
122 """
123 db = self.app.get_db_handler()
124 batch_id = db.next_counter_value(session, 'batch_id')
125 if as_str:
126 return f'{batch_id:08d}'
127 return batch_id
129 def init_batch(self, batch, session=None, progress=None, **kwargs):
130 """
131 Initialize a new batch.
133 This is called automatically from :meth:`make_batch()`.
135 Default logic does nothing; subclass should override if needed.
137 .. note::
138 *Population* of the new batch should **not** happen here;
139 see instead :meth:`populate()`.
140 """
142 def get_data_path(self, batch=None, filename=None, makedirs=False):
143 """
144 Returns a path to batch data file(s).
146 This can be used to return any of the following, depending on
147 how it's called:
149 * path to root data dir for handler's :attr:`batch_type`
150 * path to data dir for specific batch
151 * path to specific filename, for specific batch
153 For instance::
155 # nb. assuming batch_type = 'inventory'
156 batch = handler.make_batch(session, created_by=user)
158 handler.get_data_path()
159 # => env/app/data/batch/inventory
161 handler.get_data_path(batch)
162 # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4
164 handler.get_data_path(batch, 'counts.csv')
165 # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4/counts.csv
167 :param batch: Optional batch instance. If specified, will
168 return path for this batch in particular. Otherwise will
169 return the "generic" path for handler's batch type.
171 :param filename: Optional filename, in context of the batch.
172 If set, the returned path will include this filename. Only
173 relevant if ``batch`` is also specified.
175 :param makedirs: Whether the folder(s) should be created, if
176 not already present.
178 :returns: Path to root data dir for handler's batch type.
179 """
180 # get root storage path
181 rootdir = self.config.get(f'{self.config.appname}.batch.storage_path')
182 if not rootdir:
183 appdir = self.app.get_appdir()
184 rootdir = os.path.join(appdir, 'data', 'batch')
186 # get path for this batch type
187 path = os.path.join(rootdir, self.batch_type)
189 # give more precise path, if batch was specified
190 if batch:
191 uuid = batch.uuid.hex
192 # nb. we use *last 2 chars* for first part of batch uuid
193 # path. this is because uuid7 is mostly sequential, so
194 # first 2 chars do not vary enough.
195 path = os.path.join(path, uuid[-2:], uuid[:-2])
197 # maybe create data dir
198 if makedirs and not os.path.exists(path):
199 os.makedirs(path)
201 # append filename if applicable
202 if batch and filename:
203 path = os.path.join(path, filename)
205 return path
207 def should_populate(self, batch):
208 """
209 Must return true or false, indicating whether the given batch
210 should be populated from initial data source(s).
212 So, true means fill the batch with data up front - by calling
213 :meth:`do_populate()` - and false means the batch will start
214 empty.
216 Default logic here always return false; subclass should
217 override if needed.
218 """
219 return False
221 def do_populate(self, batch, progress=None):
222 """
223 Populate the batch from initial data source(s).
225 This method is a convenience wrapper, which ultimately will
226 call :meth:`populate()` for the implementation logic.
228 Therefore callers should use this ``do_populate()`` method,
229 but subclass should override :meth:`populate()` instead (if
230 needed).
232 See also :meth:`should_populate()` - you should check that
233 before calling ``do_populate()``.
234 """
235 self.populate(batch, progress=progress)
237 def populate(self, batch, progress=None):
238 """
239 Populate the batch from initial data source(s).
241 It is assumed that the data source(s) to be used will be known
242 by inspecting various properties of the batch itself.
244 Subclass should override this method to provide the
245 implementation logic. It may populate some batches
246 differently based on the batch attributes, or it may populate
247 them all the same. Whatever is needed.
249 Callers should always use :meth:`do_populate()` instead of
250 calling ``populate()`` directly.
251 """
253 def make_row(self, **kwargs):
254 """
255 Make a new row for the batch. This will be an instance of
256 :attr:`~wuttjamaican.db.model.batch.BatchMixin.__row_class__`.
258 Note that the row will **not** be added to the batch; that
259 should be done with :meth:`add_row()`.
261 :returns: A new row object, which does *not* yet belong to any batch.
262 """
263 return self.model_class.__row_class__(**kwargs)
265 def add_row(self, batch, row):
266 """
267 Add the given row to the given batch.
269 This assumes a *new* row which does not yet belong to a batch,
270 as returned by :meth:`make_row()`.
272 It will add it to batch
273 :attr:`~wuttjamaican.db.model.batch.BatchMixin.rows`, call
274 :meth:`refresh_row()` for it, and update the
275 :attr:`~wuttjamaican.db.model.batch.BatchMixin.row_count`.
276 """
277 session = self.app.get_session(batch)
278 with session.no_autoflush:
279 batch.rows.append(row)
280 self.refresh_row(row)
281 batch.row_count = (batch.row_count or 0) + 1
283 def refresh_row(self, row):
284 """
285 Update the given batch row as needed, to reflect latest data.
287 This method is a bit of a catch-all in that it could be used
288 to do any of the following (etc.):
290 * fetch latest "live" data for comparison with batch input data
291 * (re-)calculate row values based on latest data
292 * set row status based on other row attributes
294 This method is called when the row is first added to the batch
295 via :meth:`add_row()` - but may be called multiple times after
296 that depending on the workflow.
297 """
299 def do_remove_row(self, row):
300 """
301 Remove a row from its batch. This will:
303 * call :meth:`remove_row()`
304 * decrement the batch
305 :attr:`~wuttjamaican.db.model.batch.BatchMixin.row_count`
306 * call :meth:`refresh_batch_status()`
308 So, callers should use ``do_remove_row()``, but subclass
309 should (usually) override :meth:`remove_row()` etc.
310 """
311 batch = row.batch
312 session = self.app.get_session(batch)
314 self.remove_row(row)
316 if batch.row_count is not None:
317 batch.row_count -= 1
319 self.refresh_batch_status(batch)
320 session.flush()
322 def remove_row(self, row):
323 """
324 Remove a row from its batch.
326 Callers should use :meth:`do_remove_row()` instead, which
327 calls this method automatically.
329 Subclass can override this method; the default logic just
330 deletes the row.
331 """
332 session = self.app.get_session(row)
333 batch = row.batch
334 batch.rows.remove(row)
335 session.delete(row)
337 def refresh_batch_status(self, batch):
338 """
339 Update the batch status as needed.
341 This method is called when some row data has changed for the
342 batch, e.g. from :meth:`do_remove_row()`.
344 It does nothing by default; subclass may override to set these
345 attributes on the batch:
347 * :attr:`~wuttjamaican.db.model.batch.BatchMixin.status_code`
348 * :attr:`~wuttjamaican.db.model.batch.BatchMixin.status_text`
349 """
351 def why_not_execute(self, batch, user=None, **kwargs):
352 """
353 Returns text indicating the reason (if any) that a given batch
354 should *not* be executed.
356 By default the only reason a batch cannot be executed, is if
357 it has already been executed. But in some cases it should be
358 more restrictive; hence this method.
360 A "brief but descriptive" message should be returned, which
361 may be displayed to the user e.g. so they understand why the
362 execute feature is not allowed for the batch. (There is no
363 need to check if batch is already executed since other logic
364 handles that.)
366 If no text is returned, the assumption will be made that this
367 batch is safe to execute.
369 :param batch: The batch in question; potentially eligible for
370 execution.
372 :param user: :class:`~wuttjamaican.db.model.auth.User` who
373 might choose to execute the batch.
375 :param \**kwargs: Execution kwargs for the batch, if known.
376 Should be similar to those for :meth:`execute()`.
378 :returns: Text reason to prevent execution, or ``None``.
380 The user interface should normally check this and if it
381 returns anything, that should be shown and the user should be
382 prevented from executing the batch.
384 However :meth:`do_execute()` will also call this method, and
385 raise a ``RuntimeError`` if text was returned. This is done
386 out of safety, to avoid relying on the user interface.
387 """
389 def describe_execution(self, batch, user=None, **kwargs):
390 """
391 This should return some text which briefly describes what will
392 happen when the given batch is executed.
394 Note that Markdown is supported here, e.g.::
396 def describe_execution(self, batch, **kwargs):
397 return \"""
399 This batch does some crazy things!
401 **you cannot possibly fathom it**
403 here are a few of them:
405 - first
406 - second
407 - third
408 \"""
410 Nothing is returned by default; subclass should define.
412 :param batch: The batch in question; eligible for execution.
414 :param user: Reference to current user who might choose to
415 execute the batch.
417 :param \**kwargs: Execution kwargs for the batch; should be
418 similar to those for :meth:`execute()`.
420 :returns: Markdown text describing batch execution.
421 """
423 def get_effective_rows(self, batch):
424 """
425 This should return a list of "effective" rows for the batch.
427 In other words, which rows should be "acted upon" when the
428 batch is executed.
430 The default logic returns the full list of batch
431 :attr:`~wuttjamaican.db.model.batch.BatchMixin.rows`, but
432 subclass may need to filter by status code etc.
433 """
434 return batch.rows
436 def do_execute(self, batch, user, progress=None, **kwargs):
437 """
438 Perform the execution steps for a batch.
440 This first calls :meth:`why_not_execute()` to make sure this
441 is even allowed.
443 If so, it calls :meth:`execute()` and then updates
444 :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed` and
445 :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed_by` on
446 the batch, to reflect current time+user.
448 So, callers should use ``do_execute()``, and subclass should
449 override :meth:`execute()`.
451 :param batch: The :term:`batch` to execute; instance of
452 :class:`~wuttjamaican.db.model.batch.BatchMixin` (among
453 other classes).
455 :param user: :class:`~wuttjamaican.db.model.auth.User` who is
456 executing the batch.
458 :param progress: Optional progress indicator factory.
460 :param \**kwargs: Additional kwargs as needed. These are
461 passed as-is to :meth:`why_not_execute()` and
462 :meth:`execute()`.
464 :returns: Whatever was returned from :meth:`execute()` - often
465 ``None``.
466 """
467 if batch.executed:
468 raise ValueError(f"batch has already been executed: {batch}")
470 reason = self.why_not_execute(batch, user=user, **kwargs)
471 if reason:
472 raise RuntimeError(f"batch execution not allowed: {reason}")
474 result = self.execute(batch, user=user, progress=progress, **kwargs)
475 batch.executed = datetime.datetime.now()
476 batch.executed_by = user
477 return result
479 def execute(self, batch, user=None, progress=None, **kwargs):
480 """
481 Execute the given batch.
483 Callers should use :meth:`do_execute()` instead, which calls
484 this method automatically.
486 This does nothing by default; subclass must define logic.
488 :param batch: A :term:`batch`; instance of
489 :class:`~wuttjamaican.db.model.batch.BatchMixin` (among
490 other classes).
492 :param user: :class:`~wuttjamaican.db.model.auth.User` who is
493 executing the batch.
495 :param progress: Optional progress indicator factory.
497 :param \**kwargs: Additional kwargs which may affect the batch
498 execution behavior. There are none by default, but some
499 handlers may declare/use them.
501 :returns: ``None`` by default, but subclass can return
502 whatever it likes, in which case that will be also returned
503 to the caller from :meth:`do_execute()`.
504 """
506 def do_delete(self, batch, user, dry_run=False, progress=None, **kwargs):
507 """
508 Delete the given batch entirely.
510 This will delete the batch proper, all data rows, and any
511 files which may be associated with it.
512 """
513 session = self.app.get_session(batch)
515 # remove data files
516 path = self.get_data_path(batch)
517 if os.path.exists(path) and not dry_run:
518 shutil.rmtree(path)
520 # remove batch proper
521 session.delete(batch)