NCBI C++ ToolKit
nc_storage_blob.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef NETCACHE__NC_STORAGE_BLOB__HPP
2 #define NETCACHE__NC_STORAGE_BLOB__HPP
3 /* $Id: nc_storage_blob.hpp 77868 2017-05-11 13:15:22Z gouriano $
4  * ===========================================================================
5  *
6  * PUBLIC DOMAIN NOTICE
7  * National Center for Biotechnology Information
8  *
9  * This software/database is a "United States Government Work" under the
10  * terms of the United States Copyright Act. It was written as part of
11  * the author's official duties as a United States Government employee and
12  * thus cannot be copyrighted. This software/database is freely available
13  * to the public for use. The National Library of Medicine and the U.S.
14  * Government have not placed any restriction on its use or reproduction.
15  *
16  * Although all reasonable efforts have been taken to ensure the accuracy
17  * and reliability of the software and data, the NLM and the U.S.
18  * Government do not and cannot warrant the performance or results that
19  * may be obtained by using this software or data. The NLM and the U.S.
20  * Government disclaim all warranties, express or implied, including
21  * warranties of performance, merchantability or fitness for any particular
22  * purpose.
23  *
24  * Please cite the author in any work or product based on this material.
25  *
26  * ===========================================================================
27  *
28  * Authors: Pavel Ivanov
29  */
30 
31 
32 #include "nc_db_info.hpp"
33 
34 
36 
37 
38 class CNCBlobStorage;
39 class CNCBlobAccessor;
40 class SNCCacheData;
41 class CCurVerReader;
42 struct SNCStateStat;
43 
44 
45 class CNCBlobVerManager : public CObject, public CSrvTask
46 {
47 public:
48  static CNCBlobVerManager* Get(Uint2 time_bucket,
49  const string& key,
50  SNCCacheData* cache_data,
51  bool for_new_version);
52  void ObtainReference(void);
53  void Release(void);
54 
55  void RequestCurVersion(CSrvTransConsumer* consumer);
58 
59  void FinalizeWriting(SNCBlobVerData* ver_data);
60  void DeadTimeChanged(SNCBlobVerData* ver_data);
61  void DeleteVersion(const SNCBlobVerData* ver_data);
62  void DeleteDeadVersion(int cut_time);
63 
65 
66 public:
67  void RevokeDataWrite();
68  void DataWritten(void);
69  void RequestMemRelease(void);
70  const string& GetKey(void);
71 
72 private:
75  CNCBlobVerManager(void);
76 
77  CNCBlobVerManager(Uint2 time_bucket,
78  const string& key,
79  SNCCacheData* cache_data);
80  virtual ~CNCBlobVerManager(void);
81 
82  virtual void DeleteThis(void);
83  virtual void ExecuteSlice(TSrvThreadNum thr_num);
84 
85  void x_DeleteCurVersion(void);
86  void x_ReleaseMgr(void);
87 
88 
89  friend class CCurVerReader;
90 
91 
97  string m_Key;
99 };
100 
101 
103 {
104 public:
105  void RequestMetaInfo(CSrvTask* owner);
106  bool IsMetaInfoReady(void);
107  /// Check if blob exists.
108  /// Method can be called only after lock is acquired.
109  bool IsBlobExists(void) const;
110  /// Check if password provided for accessing the blob was correct
111  bool IsAuthorized(void) const;
112  bool HasError(void) const;
113  /// Get key of the blob.
114  /// Method can be called only after lock is acquired.
115  const string& GetBlobKey (void) const;
116  Uint2 GetTimeBucket (void) const;
117  /// Get size of the blob.
118  /// Method can be called only after lock is acquired.
119  /// If blob doesn't exist then value is undefined.
120  Uint8 GetCurBlobSize (void) const;
121  Uint8 GetNewBlobSize (void) const;
122  Uint8 GetSizeRead (void) const;
123  /// Check if blob is already expired but not yet deleted by GC.
124  /// Method can be called only after lock is acquired.
125  /// If blob doesn't exist then value is undefined.
126  bool IsCurBlobExpired (void) const;
127  bool IsCurVerExpired (void) const;
128  bool IsCurBlobDead (void) const;
129  /// Get type of access this holder was created for
130  ENCAccessType GetAccessType (void) const;
131 
132  /// Initially set current position in the blob to start reading from
133  void SetPosition(Uint8 pos);
134  Uint8 GetPosition(void);
135  Uint4 GetReadMemSize(void);
136  const void* GetReadMemPtr(void);
137  void MoveReadPos(Uint4 move_size);
138  unsigned int GetCurBlobTTL(void) const;
139  unsigned int GetNewBlobTTL(void) const;
140  /// Set blob's timeout after last access before it will be deleted.
141  /// Method can be called only after lock is acquired, if blob exists and
142  /// lock is acquired for eNCDelete or eNCCreate access.
143  void SetBlobTTL(unsigned int ttl);
144  unsigned int GetCurVersionTTL(void) const;
145  unsigned int GetNewVersionTTL(void) const;
146  void SetVersionTTL(int ttl);
147  int GetCurBlobVersion(void) const;
148  void SetBlobVersion(int ver);
149  Uint8 GetCurBlobCreateTime(void) const;
150  Uint8 GetNewBlobCreateTime(void) const;
151  void SetBlobCreateTime(Uint8 create_time);
152  int GetCurBlobDeadTime(void) const;
153  int GetCurBlobExpire(void) const;
154  int GetNewBlobExpire(void) const;
155  int GetCurVerExpire(void) const;
156  void SetCurBlobExpire(int expire, int dead_time = 0);
157  void SetNewBlobExpire(int expire, int dead_time = 0);
158  void SetCurVerExpire(int dead_time);
159  void SetNewVerExpire(int dead_time);
160  Uint8 GetCurCreateServer(void) const;
161  Uint8 GetNewCreateServer(void) const;
162  Uint4 GetCurCreateId(void) const;
163  void SetCreateServer(Uint8 create_server, Uint4 create_id);
164  void UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time);
165  bool IsValid(void) const;
166  Uint8 GetValidServer(void) const;
167  string GetCurPassword(void) const;
170 
171  size_t GetWriteMemSize(void);
172  void* GetWriteMemPtr(void);
173  void MoveWritePos(Uint4 move_size);
174  void Finalize(void);
175 
176  /// Release blob lock.
177  /// No other method can be called after call to this one.
178  void Release(void);
179 
180 public:
181  // For internal use only
182 
183  /// Create holder of blob lock bound to the given NetCache storage.
184  /// Lock holder is yet not usable after construction until Prepare()
185  /// and InitializeLock() are called.
186  CNCBlobAccessor(void);
187  virtual ~CNCBlobAccessor(void);
188 
189  bool IsPurged(const CNCBlobKeyLight& nc_key) const;
190  static Uint8 GetPurgeCount();
191  static bool Purge(const CNCBlobKeyLight& nc_key, Uint8 when);
192  static string GetPurgeData(char separator='\n');
193  static bool UpdatePurgeData(const string& data, char separator='\n');
194 
195  static void SetFailedWriteCount(int failed_write);
196  static int GetFailedWriteCount(void);
197  static void PutFailed(const string& blob_key);
198  static void PutSucceeded(const string& blob_key);
199  static bool HasPutSucceeded(const string& blob_key);
200 
201  /// Prepare lock for the blob identified by key, subkey and version.
202  /// Method only initializes necessary variables, actual acquiring of the
203  /// lock happens in InitializeLock() method.
204  /// If access is eNCRead and storage's IsChangeTimeOnRead() returns TRUE
205  /// then holder will change blob's access time when lock is released. If
206  /// access is eNCCreate then blob will be automatically deleted
207  /// when lock is released if blob wasn't properly finalized.
208  ///
209  /// @param key
210  /// Key of the blob
211  /// @param access
212  /// Required access to the blob
213  void Prepare(const string& key,
214  const string& password,
215  Uint2 time_bucket,
216  ENCAccessType access_type);
217  /// Initialize and acquire the lock.
218  /// Should be called only after Prepare().
219  void Initialize(SNCCacheData* cache_data);
220  void Deinitialize(void);
221 
222 private:
225 
226  virtual void ExecuteSlice(TSrvThreadNum thr_num);
227 
228  void x_CreateNewData(void);
229  void x_DelCorruptedVersion(void);
230 
231 
232  /// Type of access requested for the blob
234  string m_BlobKey;
235  /// Password that was used for access to the blob
236  string m_Password;
246  /// Current position of reading/writing inside blob's chunk
250  char* m_Buffer;
252 };
253 
254 
256 {
257 public:
259  virtual ~CCurVerReader(void);
260 
261 private:
262  virtual void ExecuteSlice(TSrvThreadNum thr_num);
263 
264 
266 };
267 
268 
270 {
271 public:
272  static void Initialize(void);
273  static void ReadState(SNCStateStat& state);
274 
275  // statistics
276  static void AnotherServerMain(void);
277  static void StartSyncBlob(Uint8 create_time);
278  static void RecordNotifyUpdateBlob(Uint8 update_received);
279  static void ResetStatCounters(void);
280 
281 private:
282  CWriteBackControl(void);
283  virtual ~CWriteBackControl(void);
284 
285  virtual void ExecuteSlice(TSrvThreadNum thr_num);
286 };
287 
288 
291 int GetWBWriteTimeout(void);
292 int GetWBFailedWriteDelay(void);
293 
294 void SetWBSoftSizeLimit(Uint8 limit);
295 void SetWBHardSizeLimit(Uint8 limit);
296 void SetWBWriteTimeout(int timeout1, int timeout2);
297 void SetWBFailedWriteDelay(int delay);
298 void SetWBInitialSyncComplete(void);
299 
300 
302 {
303 public:
304  CWBMemDeleter(char* mem, Uint4 mem_size);
305  virtual ~CWBMemDeleter(void);
306 
307 private:
308  virtual void ExecuteRCU(void);
309 
310 
311  char* m_Mem;
313 };
314 
315 
316 
317 //////////////////////////////////////////////////////////////////////////
318 // Inline methods
319 //////////////////////////////////////////////////////////////////////////
320 
321 inline SNCCacheData*
323 {
324  return m_CacheData;
325 }
326 
327 inline const string&
329 {
330  return m_Key;
331 }
332 
333 inline void
335 {
337 }
338 
339 
340 inline bool
342 {
343  return m_MetaInfoReady;
344 }
345 
346 inline bool
348 {
349  return m_CurData.NotNull();
350 }
351 
352 inline bool
354 {
355  return !IsBlobExists() || IsCurBlobExpired()
357 }
358 
359 inline bool
361 {
362  return m_HasError;
363 }
364 
365 inline const string&
367 {
368  return m_BlobKey;
369 }
370 
371 inline Uint2
373 {
374  return m_TimeBucket;
375 }
376 
377 inline Uint8
379 {
380  return m_CurData->size;
381 }
382 
383 inline Uint8
385 {
386  return m_NewData? m_NewData->size: 0;
387 }
388 
389 inline Uint8
391 {
392  return m_SizeRead;
393 }
394 
395 inline int
397 {
398  return m_CurData->blob_ver;
399 }
400 
401 inline unsigned int
403 {
404  return m_CurData->ttl;
405 }
406 
407 inline unsigned int
409 {
410  return m_NewData->ttl;
411 }
412 
413 inline bool
415 {
416  return m_CurData->expire <= CSrvTime::CurSecs();
417 }
418 
419 inline bool
421 {
423 }
424 inline bool
426 {
428 }
429 
430 inline void
432 {
433  x_CreateNewData();
434  Uint8 prev = GetCurBlobCreateTime();
435  if (create_time < prev) {
436 // new data cannot be created earlier than the current one.
437 // due to time differences between servers, or who knows what else..
438 // anyway, check here
439  create_time = prev + 1;
440  }
441  m_NewData->create_time = create_time;
442 }
443 
444 inline Uint8
446 {
447  return m_CurData.NotNull()? m_CurData->create_time: 0;
448 }
449 
450 inline Uint8
452 {
453  return m_NewData->create_time;
454 }
455 
456 inline int
458 {
459  return m_CurData->dead_time;
460 }
461 
462 inline int
464 {
465  return m_CurData->expire;
466 }
467 
468 inline int
470 {
471  return m_NewData->expire;
472 }
473 
474 inline int
476 {
477  return m_CurData->ver_expire;
478 }
479 
480 inline void
481 CNCBlobAccessor::SetNewBlobExpire(int expire, int dead_time /* = 0 */)
482 {
483  x_CreateNewData();
484  m_NewData->expire = expire;
485  int d_time = max(dead_time, expire + 40);
486  if (dead_time == 0 && m_CurData) {
487  d_time = max(m_CurData->dead_time, d_time);
488  }
489  m_NewData->dead_time = d_time;
490 }
491 
492 inline unsigned int
494 {
495  return m_CurData->ver_ttl;
496 }
497 
498 inline unsigned int
500 {
501  return m_NewData->ver_ttl;
502 }
503 
504 inline void
506 {
507  x_CreateNewData();
508  m_NewData->ver_ttl = ttl;
509 }
510 
511 inline void
513 {
514  x_CreateNewData();
515  m_NewData->ttl = ttl;
516 }
517 
518 inline void
520 {
521  x_CreateNewData();
522  m_NewData->blob_ver = ver;
523 }
524 
525 inline void
526 CNCBlobAccessor::SetCurBlobExpire(int expire, int dead_time /* = 0 */)
527 {
528  m_CurData->expire = expire;
529  m_CurData->dead_time = max(expire + 40, max(m_CurData->dead_time, dead_time));
531 }
532 
533 inline void
535 {
536  m_CurData->ver_expire = expire;
538 }
539 
540 inline void
542 {
543  x_CreateNewData();
544  m_NewData->ver_expire = expire;
545 }
546 
547 inline void
549  Uint4 create_id)
550 {
551  x_CreateNewData();
552  m_NewData->create_server = create_server;
553  m_NewData->create_id = create_id;
554 }
555 inline void
557 {
558  if (m_CurData->create_time < upd_time) {
559  m_CurData->updated_on_server = upd_server;
560  m_CurData->updated_at_time = upd_time;
562  }
563 }
564 
565 inline bool
567 {
568  return m_CurData.NotNull()? (m_CurData->updated_on_server == 0) : true;
569 }
570 
571 inline Uint8
573 {
575 }
576 
577 
578 inline Uint8
580 {
582 }
583 
584 inline Uint8
586 {
587  return m_NewData->create_server;
588 }
589 
590 inline Uint4
592 {
593  return m_CurData.NotNull()? m_CurData->create_id: 0;
594 }
595 
596 inline void
598 {
599  x_CreateNewData();
601 }
602 
603 inline ENCAccessType
605 {
606  return m_AccessType;
607 }
608 
609 inline void
611 {
613  m_ChunkPos = size_t(pos % m_CurData->chunk_size);
614 }
615 
616 inline Uint8
618 {
620 }
621 
622 inline const void*
624 {
625  return m_Buffer + m_ChunkPos;
626 }
627 
628 inline void*
630 {
631  return m_Buffer + m_ChunkPos;
632 }
633 
634 inline void
636 {
637  m_ChunkPos += move_size;
638  m_NewData->size += move_size;
639 }
640 
642 
643 #endif /* NETCACHE__NC_STORAGE_BLOB__HPP */
static void Initialize(void)
void SetPosition(Uint8 pos)
Initially set current position in the blob to start reading from.
Uint8 AsUSec(void) const
Converts object's value to microseconds since epoch.
CSrvRef< SNCBlobVerData > CreateNewVersion(void)
Uint8 GetNewBlobCreateTime(void) const
size_t GetWriteMemSize(void)
CWBMemDeleter(char *mem, Uint4 mem_size)
void SetBlobTTL(unsigned int ttl)
Set blob's timeout after last access before it will be deleted.
bool IsBlobExists(void) const
Check if blob exists.
Uint4 GetCurCreateId(void) const
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
static void ResetStatCounters(void)
SNCCacheData * m_CacheData
void SetCurBlobExpire(int expire, int dead_time=0)
Uint8 GetNewBlobSize(void) const
Main working entity in TaskServer.
Definition: srv_tasks.hpp:87
static void new_info(pcre *re, pcre_extra *study, int option, void *ptr)
Definition: pcretest.c:636
unsigned int GetCurVersionTTL(void) const
unsigned NCBI_INT8_TYPE Uint8
Unsigned 8 byte sized integer.
Definition: ncbitype.h:146
CTempString implements a light-weight string on top of a storage buffer whose lifetime management is ...
Definition: tempstr.hpp:62
const void * GetReadMemPtr(void)
unsigned int Uint4
Alias for unsigned int.
Definition: ncbitype.h:121
Uint8 update_received
Definition: nc_db_info.hpp:107
const struct ncbi::grid::netcache::search::fields::KEY key
Uint8 GetCurBlobSize(void) const
Get size of the blob.
static bool UpdatePurgeData(const string &data, char separator='\n')
static void PutSucceeded(const string &blob_key)
CSrvRef< SNCBlobVerData > GetCurVersion(void)
static string GetPurgeData(char separator='\n')
Uint8 GetCurCreateServer(void) const
virtual ~CCurVerReader(void)
int GetCurBlobDeadTime(void) const
CNCBlobVerManager * m_VerMgr
void FinalizeWriting(SNCBlobVerData *ver_data)
Full information about NetCache blob (excluding key)
Definition: nc_db_info.hpp:93
void SetCurVerExpire(int dead_time)
ENCAccessType
Type of access to NetCache blob.
Definition: nc_utils.hpp:81
void SetPassword(CTempString password)
static const unsigned char password[6][32]
Definition: pkcs5.c:305
int GetCurBlobVersion(void) const
Uint2 TSrvThreadNum
Type for thread number in TaskServer.
Definition: task_server.hpp:42
bool IsCurBlobExpired(void) const
Check if blob is already expired but not yet deleted by GC.
static void PutFailed(const string &blob_key)
Uint4 m_ChunkPos
Current position of reading/writing inside blob's chunk.
int GetCurBlobExpire(void) const
void SetVersionTTL(int ttl)
void ObtainReference(void)
void MoveReadPos(Uint4 move_size)
CNCBlobAccessor(void)
Create holder of blob lock bound to the given NetCache storage.
bool HasError(void) const
virtual void DeleteThis(void)
Virtual method "deleting" this object.
void DeleteDeadVersion(int cut_time)
void x_CreateNewData(void)
ENCAccessType m_AccessType
Type of access requested for the blob.
string m_Password
Password that was used for access to the blob.
string GetCurPassword(void) const
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:101
CCurVerReader(CNCBlobVerManager *mgr)
CNCBlobVerManager & operator=(const CNCBlobVerManager &)
CSrvRef< SNCBlobVerData > m_CurVersion
Uint2 GetTimeBucket(void) const
void SetWBHardSizeLimit(Uint8 limit)
Uint4 GetReadMemSize(void)
void SetWBInitialSyncComplete(void)
void DeleteVersion(const SNCBlobVerData *ver_data)
bool ReplaceBlobInfo(const SNCBlobVerData &new_info)
void SetNewVerExpire(int dead_time)
bool IsAuthorized(void) const
Check if password provided for accessing the blob was correct.
bool NotNull(void) const THROWS_NONE
Check if pointer is not null – same effect as NotEmpty().
Definition: ncbiobj.hpp:727
SNCCacheData * GetCacheData(void)
void RequestMetaInfo(CSrvTask *owner)
unsigned int GetCurBlobTTL(void) const
static void StartSyncBlob(Uint8 create_time)
Uint8 updated_on_server
Definition: nc_db_info.hpp:105
virtual ~CNCBlobVerManager(void)
void SetBlobCreateTime(Uint8 create_time)
static Uint8 GetPurgeCount()
void MoveWritePos(Uint4 move_size)
ENCAccessType GetAccessType(void) const
Get type of access this holder was created for.
void x_DelCorruptedVersion(void)
unsigned int GetNewVersionTTL(void) const
bool IsCurBlobDead(void) const
SNCChunkMaps * m_ChunkMaps
Uint8 GetNewCreateServer(void) const
Special task that has some information to give to its consumers but first this information should be ...
Definition: srv_tasks.hpp:379
void RequestMemRelease(void)
void SetCreateServer(Uint8 create_server, Uint4 create_id)
Blob storage for NetCache.
Definition: nc_storage.hpp:96
void DeadTimeChanged(SNCBlobVerData *ver_data)
static CNCBlobVerManager * Get(Uint2 time_bucket, const string &key, SNCCacheData *cache_data, bool for_new_version)
virtual void ExecuteRCU(void)
Method implementing RCU job that was scheduled earlier by CallRCU().
void SetNewBlobExpire(int expire, int dead_time=0)
CSrvRef< SNCBlobVerData > m_CurData
void x_DeleteCurVersion(void)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
T max(T x_, T y_)
static CSrvTime Current(void)
Exact current time with precision up to nanoseconds.
static void AnotherServerMain(void)
int GetWBWriteTimeout(void)
CNCBlobAccessor & operator=(const CNCBlobAccessor &)
void SetWBFailedWriteDelay(int delay)
Uint8 GetSizeRead(void) const
bool IsCurVerExpired(void) const
Class incorporating convenient methods to work with struct timespec.
Definition: srv_time.hpp:59
void RequestTransition(CSrvTransConsumer *consumer)
Requests task's state transition with the provided consumer to be notified when transition is finishe...
Definition: srv_tasks.cpp:167
CSrvRef< SNCBlobVerData > m_NewData
Uint8 GetWBHardSizeLimit(void)
CNCBlobVerManager * m_VerManager
static int GetFailedWriteCount(void)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
virtual ~CNCBlobAccessor(void)
Uint8 GetCurBlobCreateTime(void) const
bool IsMetaInfoReady(void)
void Prepare(const string &key, const string &password, Uint2 time_bucket, ENCAccessType access_type)
Prepare lock for the blob identified by key, subkey and version.
Consumer of notification about transition completeness in CSrvTransitionTask.
Definition: srv_tasks.hpp:422
void SetWBSoftSizeLimit(Uint8 limit)
CObject –.
Definition: ncbiobj.hpp:180
int GetWBFailedWriteDelay(void)
CCurVerReader * m_CurVerReader
static bool HasPutSucceeded(const string &blob_key)
void UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time)
bool IsValid(void) const
int GetCurVerExpire(void) const
void SetBlobVersion(int ver)
Class to derive from to use RCU mechanism.
Definition: task_server.hpp:74
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
static int CurSecs(void)
Current time in seconds since epoch (time_t).
Uint8 GetValidServer(void) const
unsigned short Uint2
Alias for unsigned short.
Definition: ncbitype.h:119
virtual ~CWriteBackControl(void)
virtual ~CWBMemDeleter(void)
static bool Purge(const CNCBlobKeyLight &nc_key, Uint8 when)
const string & GetBlobKey(void) const
Get key of the blob.
static void ReadState(SNCStateStat &state)
void RequestCurVersion(CSrvTransConsumer *consumer)
int GetNewBlobExpire(void) const
const string & GetKey(void)
bool IsPurged(const CNCBlobKeyLight &nc_key) const
void SetWBWriteTimeout(int timeout1, int timeout2)
unsigned int ttl
Definition: nc_db_info.hpp:108
void Release(void)
Release blob lock.
void Initialize(SNCCacheData *cache_data)
Initialize and acquire the lock.
static void SetFailedWriteCount(int failed_write)
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:98
Uint8 GetWBSoftSizeLimit(void)
void * GetWriteMemPtr(void)
Uint8 GetPosition(void)
Uint8 updated_at_time
Definition: nc_db_info.hpp:106
static void RecordNotifyUpdateBlob(Uint8 update_received)
unsigned int GetNewBlobTTL(void) const
Modified on Sat Sep 23 14:31:13 2017 by modify_doxy.py rev. 546573