MISR Toolkit  1.5.1
H5ACpublic.h
Go to the documentation of this file.
1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2  * Copyright by The HDF Group. *
3  * Copyright by the Board of Trustees of the University of Illinois. *
4  * All rights reserved. *
5  * *
6  * This file is part of HDF5. The full HDF5 copyright notice, including *
7  * terms governing use, modification, and redistribution, is contained in *
8  * the COPYING file, which can be found at the root of the source code *
9  * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
10  * If you do not have access to either file, you may request a copy from *
11  * help@hdfgroup.org. *
12  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13 
14 /*-------------------------------------------------------------------------
15  *
16  * Created: H5ACpublic.h
17  * Jul 10 1997
18  * Robb Matzke <matzke@llnl.gov>
19  *
20  * Purpose: Public include file for cache functions.
21  *
22  * Modifications:
23  *
24  *-------------------------------------------------------------------------
25  */
26 #ifndef _H5ACpublic_H
27 #define _H5ACpublic_H
28 
29 /* Public headers needed by this file */
30 #include "H5public.h"
31 #include "H5Cpublic.h"
32 
33 #ifdef __cplusplus
34 extern "C" {
35 #endif
36 
37 /****************************************************************************
38  *
39  * structure H5AC_cache_config_t
40  *
41  * H5AC_cache_config_t is a public structure intended for use in public APIs.
42  * At least in its initial incarnation, it is basically a copy of struct
43  * H5C_auto_size_ctl_t, minus the report_fcn field, and plus the
44  * dirty_bytes_threshold field.
45  *
46  * The report_fcn field is omitted, as including it would require us to
47  * make H5C_t structure public.
48  *
49  * The dirty_bytes_threshold field does not appear in H5C_auto_size_ctl_t,
50  * as synchronization between caches on different processes is handled at
51  * the H5AC level, not at the level of H5C. Note however that there is
52  * considerable interaction between this value and the other fields in this
53  * structure.
54  *
55  * Similarly, the open_trace_file, close_trace_file, and trace_file_name
56  * fields do not appear in H5C_auto_size_ctl_t, as most trace file
57  * issues are handled at the H5AC level. The one exception is storage of
58  * the pointer to the trace file, which is handled by H5C.
59  *
60  * The structure is in H5ACpublic.h as we may wish to allow different
61  * configuration options for metadata and raw data caches.
62  *
63  * The fields of the structure are discussed individually below:
64  *
65  * version: Integer field containing the version number of this version
66  * of the H5AC_cache_config_t structure. Any instance of
67  * H5AC_cache_config_t passed to the cache must have a known
68  * version number, or an error will be flagged.
69  *
70  * rpt_fcn_enabled: Boolean field used to enable and disable the default
71  * reporting function. This function is invoked every time the
72  * automatic cache resize code is run, and reports on its activities.
73  *
74  * This is a debugging function, and should normally be turned off.
75  *
76  * open_trace_file: Boolean field indicating whether the trace_file_name
77  * field should be used to open a trace file for the cache.
78  *
79  * The trace file is a debugging feature that allow the capture of
80  * top level metadata cache requests for purposes of debugging and/or
81  * optimization. This field should normally be set to FALSE, as
82  * trace file collection imposes considerable overhead.
83  *
84  * This field should only be set to TRUE when the trace_file_name
85  * contains the full path of the desired trace file, and either
86  * there is no open trace file on the cache, or the close_trace_file
87  * field is also TRUE.
88  *
89  * close_trace_file: Boolean field indicating whether the current trace
90  * file (if any) should be closed.
91  *
92  * See the above comments on the open_trace_file field. This field
93  * should be set to FALSE unless there is an open trace file on the
94  * cache that you wish to close.
95  *
96  * trace_file_name: Full path of the trace file to be opened if the
97  * open_trace_file field is TRUE.
98  *
99  * In the parallel case, an ascii representation of the mpi rank of
100  * the process will be appended to the file name to yield a unique
101  * trace file name for each process.
102  *
103  * The length of the path must not exceed H5AC__MAX_TRACE_FILE_NAME_LEN
104  * characters.
105  *
106  * evictions_enabled: Boolean field used to either report the current
107  * evictions enabled status of the cache, or to set the cache's
108  * evictions enabled status.
109  *
110  * In general, the metadata cache should always be allowed to
111  * evict entries. However, in some cases it is advantageous to
112  * disable evictions briefly, and thereby postpone metadata
113  * writes. However, this must be done with care, as the cache
114  * can grow quickly. If you do this, re-enable evictions as
115  * soon as possible and monitor cache size.
116  *
117  * At present, evictions can only be disabled if automatic
118  * cache resizing is also disabled (that is, ( incr_mode ==
119  * H5C_incr__off ) && ( decr_mode == H5C_decr__off )). There
120  * is no logical reason why this should be so, but it simplifies
121  * implementation and testing, and I can't think of any reason
122  * why it would be desireable. If you can think of one, I'll
123  * revisit the issue.
124  *
125  * set_initial_size: Boolean flag indicating whether the size of the
126  * initial size of the cache is to be set to the value given in
127  * the initial_size field. If set_initial_size is FALSE, the
128  * initial_size field is ignored.
129  *
130  * initial_size: If enabled, this field contain the size the cache is
131  * to be set to upon receipt of this structure. Needless to say,
132  * initial_size must lie in the closed interval [min_size, max_size].
133  *
134  * min_clean_fraction: double in the range 0 to 1 indicating the fraction
135  * of the cache that is to be kept clean. This field is only used
136  * in parallel mode. Typical values are 0.1 to 0.5.
137  *
138  * max_size: Maximum size to which the cache can be adjusted. The
139  * supplied value must fall in the closed interval
140  * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, max_size must
141  * be greater than or equal to min_size.
142  *
143  * min_size: Minimum size to which the cache can be adjusted. The
144  * supplied value must fall in the closed interval
145  * [H5C__MIN_MAX_CACHE_SIZE, H5C__MAX_MAX_CACHE_SIZE]. Also, min_size
146  * must be less than or equal to max_size.
147  *
148  * epoch_length: Number of accesses on the cache over which to collect
149  * hit rate stats before running the automatic cache resize code,
150  * if it is enabled.
151  *
152  * At the end of an epoch, we discard prior hit rate data and start
153  * collecting afresh. The epoch_length must lie in the closed
154  * interval [H5C__MIN_AR_EPOCH_LENGTH, H5C__MAX_AR_EPOCH_LENGTH].
155  *
156  *
157  * Cache size increase control fields:
158  *
159  * incr_mode: Instance of the H5C_cache_incr_mode enumerated type whose
160  * value indicates how we determine whether the cache size should be
161  * increased. At present there are two possible values:
162  *
163  * H5C_incr__off: Don't attempt to increase the size of the cache
164  * automatically.
165  *
166  * When this increment mode is selected, the remaining fields
167  * in the cache size increase section ar ignored.
168  *
169  * H5C_incr__threshold: Attempt to increase the size of the cache
170  * whenever the average hit rate over the last epoch drops
171  * below the value supplied in the lower_hr_threshold
172  * field.
173  *
174  * Note that this attempt will fail if the cache is already
175  * at its maximum size, or if the cache is not already using
176  * all available space.
177  *
178  * Note that you must set decr_mode to H5C_incr__off if you
179  * disable metadata cache entry evictions.
180  *
181  * lower_hr_threshold: Lower hit rate threshold. If the increment mode
182  * (incr_mode) is H5C_incr__threshold and the hit rate drops below the
183  * value supplied in this field in an epoch, increment the cache size by
184  * size_increment. Note that cache size may not be incremented above
185  * max_size, and that the increment may be further restricted by the
186  * max_increment field if it is enabled.
187  *
188  * When enabled, this field must contain a value in the range [0.0, 1.0].
189  * Depending on the incr_mode selected, it may also have to be less than
190  * upper_hr_threshold.
191  *
192  * increment: Double containing the multiplier used to derive the new
193  * cache size from the old if a cache size increment is triggered.
194  * The increment must be greater than 1.0, and should not exceed 2.0.
195  *
196  * The new cache size is obtained my multiplying the current max cache
197  * size by the increment, and then clamping to max_size and to stay
198  * within the max_increment as necessary.
199  *
200  * apply_max_increment: Boolean flag indicating whether the max_increment
201  * field should be used to limit the maximum cache size increment.
202  *
203  * max_increment: If enabled by the apply_max_increment field described
204  * above, this field contains the maximum number of bytes by which the
205  * cache size can be increased in a single re-size.
206  *
207  * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated
208  * type whose value indicates whether and by which algorithm we should
209  * make flash increases in the size of the cache to accommodate insertion
210  * of large entries and large increases in the size of a single entry.
211  *
212  * The addition of the flash increment mode was occasioned by performance
213  * problems that appear when a local heap is increased to a size in excess
214  * of the current cache size. While the existing re-size code dealt with
215  * this eventually, performance was very bad for the remainder of the
216  * epoch.
217  *
218  * At present, there are two possible values for the flash_incr_mode:
219  *
220  * H5C_flash_incr__off: Don't perform flash increases in the size of
221  * the cache.
222  *
223  * H5C_flash_incr__add_space: Let x be either the size of a newly
224  * newly inserted entry, or the number of bytes by which the
225  * size of an existing entry has been increased.
226  *
227  * If
228  * x > flash_threshold * current max cache size,
229  *
230  * increase the current maximum cache size by x * flash_multiple
231  * less any free space in the cache, and star a new epoch. For
232  * now at least, pay no attention to the maximum increment.
233  *
234  * In both of the above cases, the flash increment pays no attention to
235  * the maximum increment (at least in this first incarnation), but DOES
236  * stay within max_size.
237  *
238  * With a little thought, it should be obvious that the above flash
239  * cache size increase algorithm is not sufficient for all circumstances
240  * -- for example, suppose the user round robins through
241  * (1/flash_threshold) +1 groups, adding one data set to each on each
242  * pass. Then all will increase in size at about the same time, requiring
243  * the max cache size to at least double to maintain acceptable
244  * performance, however the above flash increment algorithm will not be
245  * triggered.
246  *
247  * Hopefully, the add space algorithms detailed above will be sufficient
248  * for the performance problems encountered to date. However, we should
249  * expect to revisit the issue.
250  *
251  * flash_multiple: Double containing the multiple described above in the
252  * H5C_flash_incr__add_space section of the discussion of the
253  * flash_incr_mode section. This field is ignored unless flash_incr_mode
254  * is H5C_flash_incr__add_space.
255  *
256  * flash_threshold: Double containing the factor by which current max cache
257  * size is multiplied to obtain the size threshold for the add_space flash
258  * increment algorithm. The field is ignored unless flash_incr_mode is
259  * H5C_flash_incr__add_space.
260  *
261  *
262  * Cache size decrease control fields:
263  *
264  * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose
265  * value indicates how we determine whether the cache size should be
266  * decreased. At present there are four possibilities.
267  *
268  * H5C_decr__off: Don't attempt to decrease the size of the cache
269  * automatically.
270  *
271  * When this increment mode is selected, the remaining fields
272  * in the cache size decrease section are ignored.
273  *
274  * H5C_decr__threshold: Attempt to decrease the size of the cache
275  * whenever the average hit rate over the last epoch rises
276  * above the value supplied in the upper_hr_threshold
277  * field.
278  *
279  * H5C_decr__age_out: At the end of each epoch, search the cache for
280  * entries that have not been accessed for at least the number
281  * of epochs specified in the epochs_before_eviction field, and
282  * evict these entries. Conceptually, the maximum cache size
283  * is then decreased to match the new actual cache size. However,
284  * this reduction may be modified by the min_size, the
285  * max_decrement, and/or the empty_reserve.
286  *
287  * H5C_decr__age_out_with_threshold: Same as age_out, but we only
288  * attempt to reduce the cache size when the hit rate observed
289  * over the last epoch exceeds the value provided in the
290  * upper_hr_threshold field.
291  *
292  * Note that you must set decr_mode to H5C_decr__off if you
293  * disable metadata cache entry evictions.
294  *
295  * upper_hr_threshold: Upper hit rate threshold. The use of this field
296  * varies according to the current decr_mode:
297  *
298  * H5C_decr__off or H5C_decr__age_out: The value of this field is
299  * ignored.
300  *
301  * H5C_decr__threshold: If the hit rate exceeds this threshold in any
302  * epoch, attempt to decrement the cache size by size_decrement.
303  *
304  * Note that cache size may not be decremented below min_size.
305  *
306  * Note also that if the upper_threshold is 1.0, the cache size
307  * will never be reduced.
308  *
309  * H5C_decr__age_out_with_threshold: If the hit rate exceeds this
310  * threshold in any epoch, attempt to reduce the cache size
311  * by evicting entries that have not been accessed for more
312  * than the specified number of epochs.
313  *
314  * decrement: This field is only used when the decr_mode is
315  * H5C_decr__threshold.
316  *
317  * The field is a double containing the multiplier used to derive the
318  * new cache size from the old if a cache size decrement is triggered.
319  * The decrement must be in the range 0.0 (in which case the cache will
320  * try to contract to its minimum size) to 1.0 (in which case the
321  * cache will never shrink).
322  *
323  * apply_max_decrement: Boolean flag used to determine whether decrements
324  * in cache size are to be limited by the max_decrement field.
325  *
326  * max_decrement: Maximum number of bytes by which the cache size can be
327  * decreased in a single re-size. Note that decrements may also be
328  * restricted by the min_size of the cache, and (in age out modes) by
329  * the empty_reserve field.
330  *
331  * epochs_before_eviction: Integer field used in H5C_decr__age_out and
332  * H5C_decr__age_out_with_threshold decrement modes.
333  *
334  * This field contains the number of epochs an entry must remain
335  * unaccessed before it is evicted in an attempt to reduce the
336  * cache size. If applicable, this field must lie in the range
337  * [1, H5C__MAX_EPOCH_MARKERS].
338  *
339  * apply_empty_reserve: Boolean field controlling whether the empty_reserve
340  * field is to be used in computing the new cache size when the
341  * decr_mode is H5C_decr__age_out or H5C_decr__age_out_with_threshold.
342  *
343  * empty_reserve: To avoid a constant racheting down of cache size by small
344  * amounts in the H5C_decr__age_out and H5C_decr__age_out_with_threshold
345  * modes, this field allows one to require that any cache size
346  * reductions leave the specified fraction of unused space in the cache.
347  *
348  * The value of this field must be in the range [0.0, 1.0]. I would
349  * expect typical values to be in the range of 0.01 to 0.1.
350  *
351  *
352  * Parallel Configuration Fields:
353  *
354  * In PHDF5, all operations that modify metadata must be executed collectively.
355  *
356  * We used to think that this was enough to ensure consistency across the
357  * metadata caches, but since we allow processes to read metadata individually,
358  * the order of dirty entries in the LRU list can vary across processes,
359  * which can result in inconsistencies between the caches.
360  *
361  * PHDF5 uses several strategies to prevent such inconsistencies in metadata,
362  * all of which use the fact that the same stream of dirty metadata is seen
363  * by all processes for purposes of synchronization. This is done by
364  * having each process count the number of bytes of dirty metadata generated,
365  * and then running a "sync point" whenever this count exceeds a user
366  * specified threshold (see dirty_bytes_threshold below).
367  *
368  * The current metadata write strategy is indicated by the
369  * metadata_write_strategy field. The possible values of this field, along
370  * with the associated metadata write strategies are discussed below.
371  *
372  * dirty_bytes_threshold: Threshold of dirty byte creation used to
373  * synchronize updates between caches. (See above for outline and
374  * motivation.)
375  *
376  * This value MUST be consistant across all processes accessing the
377  * file. This field is ignored unless HDF5 has been compiled for
378  * parallel.
379  *
380  * metadata_write_strategy: Integer field containing a code indicating the
381  * desired metadata write strategy. The valid values of this field
382  * are enumerated and discussed below:
383  *
384  *
385  * H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY:
386  *
387  * When metadata_write_strategy is set to this value, only process
388  * zero is allowed to write dirty metadata to disk. All other
389  * processes must retain dirty metadata until they are informed at
390  * a sync point that the dirty metadata in question has been written
391  * to disk.
392  *
393  * When the sync point is reached (or when there is a user generated
394  * flush), process zero flushes sufficient entries to bring it into
395  * complience with its min clean size (or flushes all dirty entries in
396  * the case of a user generated flush), broad casts the list of
397  * entries just cleaned to all the other processes, and then exits
398  * the sync point.
399  *
400  * Upon receipt of the broadcast, the other processes mark the indicated
401  * entries as clean, and leave the sync point as well.
402  *
403  *
404  * H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED:
405  *
406  * In the distributed metadata write strategy, process zero still makes
407  * the decisions as to what entries should be flushed, but the actual
408  * flushes are distributed across the processes in the computation to
409  * the extent possible.
410  *
411  * In this strategy, when a sync point is triggered (either by dirty
412  * metadata creation or manual flush), all processes enter a barrier.
413  *
414  * On the other side of the barrier, process 0 constructs an ordered
415  * list of the entries to be flushed, and then broadcasts this list
416  * to the caches in all the processes.
417  *
418  * All processes then scan the list of entries to be flushed, flushing
419  * some, and marking the rest as clean. The algorithm for this purpose
420  * ensures that each entry in the list is flushed exactly once, and
421  * all are marked clean in each cache.
422  *
423  * Note that in the case of a flush of the cache, no message passing
424  * is necessary, as all processes have the same list of dirty entries,
425  * and all of these entries must be flushed. Thus in this case it is
426  * sufficient for each process to sort its list of dirty entries after
427  * leaving the initial barrier, and use this list as if it had been
428  * received from process zero.
429  *
430  * To avoid possible messages from the past/future, all caches must
431  * wait until all caches are done before leaving the sync point.
432  *
433  ****************************************************************************/
434 
435 #define H5AC__CURR_CACHE_CONFIG_VERSION 1
436 #define H5AC__MAX_TRACE_FILE_NAME_LEN 1024
437 
438 #define H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY 0
439 #define H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED 1
440 
441 typedef struct H5AC_cache_config_t
442 {
443  /* general configuration fields: */
444  int version;
445 
447 
451 
453 
455  size_t initial_size;
456 
458 
459  size_t max_size;
460  size_t min_size;
461 
462  long int epoch_length;
463 
464 
465  /* size increase control fields: */
467 
469 
470  double increment;
471 
474 
478 
479 
480  /* size decrease control fields: */
482 
484 
485  double decrement;
486 
489 
491 
494 
495 
496  /* parallel configuration fields: */
499 
501 
502 
503 #ifdef __cplusplus
504 }
505 #endif
506 #endif
H5C_cache_flash_incr_mode
Definition: H5Cpublic.h:42
unsigned int hbool_t
Definition: H5public.h:142
hbool_t rpt_fcn_enabled
Definition: H5ACpublic.h:446
hbool_t apply_max_increment
Definition: H5ACpublic.h:472
long int epoch_length
Definition: H5ACpublic.h:462
H5C_cache_decr_mode
Definition: H5Cpublic.h:48
hbool_t set_initial_size
Definition: H5ACpublic.h:454
#define H5AC__MAX_TRACE_FILE_NAME_LEN
Definition: H5ACpublic.h:436
double min_clean_fraction
Definition: H5ACpublic.h:457
enum H5C_cache_incr_mode incr_mode
Definition: H5ACpublic.h:466
hbool_t open_trace_file
Definition: H5ACpublic.h:448
char trace_file_name[H5AC__MAX_TRACE_FILE_NAME_LEN+1]
Definition: H5ACpublic.h:450
hbool_t apply_empty_reserve
Definition: H5ACpublic.h:492
enum H5C_cache_decr_mode decr_mode
Definition: H5ACpublic.h:481
double lower_hr_threshold
Definition: H5ACpublic.h:468
enum H5C_cache_flash_incr_mode flash_incr_mode
Definition: H5ACpublic.h:475
hbool_t apply_max_decrement
Definition: H5ACpublic.h:487
hbool_t evictions_enabled
Definition: H5ACpublic.h:452
double upper_hr_threshold
Definition: H5ACpublic.h:483
H5C_cache_incr_mode
Definition: H5Cpublic.h:36
hbool_t close_trace_file
Definition: H5ACpublic.h:449
struct H5AC_cache_config_t H5AC_cache_config_t

MISR Toolkit - Copyright © 2005 - 2020 Jet Propulsion Laboratory
Generated on Fri Jun 19 2020 22:49:52