AWS SDK for C++  1.9.109
AWS SDK for C++
TransferHandle.h
Go to the documentation of this file.
1 
6 #pragma once
7 
12 #include <aws/core/utils/UUID.h>
15 #include <aws/s3/S3Errors.h>
16 #include <iostream>
17 #include <atomic>
18 #include <mutex>
19 #include <condition_variable>
20 
21 namespace Aws
22 {
23  namespace Utils
24  {
25  template < typename T > class Array;
26  }
27 
28  namespace Transfer
29  {
30  class TransferHandle;
31 
32  typedef std::function<Aws::IOStream*(void)> CreateDownloadStreamCallback;
33 
34  static const char CLASS_TAG[] = "TransferManager";
35 
37  {
39  versionId("")
40  {}
41 
43 
44  // TBI: controls for in-memory parts vs. resumable file-based parts with state serialization to/from file
45  };
46 
47  class PartState
48  {
49  public:
51  PartState(int partId, uint64_t bestProgressInBytes, uint64_t sizeInBytes, bool lastPart = false);
52 
53  int GetPartId() const { return m_partId; }
54 
55  uint64_t GetBestProgressInBytes() const { return m_bestProgressInBytes; }
56  void SetBestProgressInBytes(uint64_t progressInBytes) { m_bestProgressInBytes = progressInBytes; }
57 
58  uint64_t GetSizeInBytes() const { return m_sizeInBytes; }
59  void SetSizeInBytes(uint64_t sizeInBytes) { m_sizeInBytes = sizeInBytes; }
60 
61  void Reset();
62 
63  void OnDataTransferred(uint64_t amount, const std::shared_ptr<TransferHandle> &transferHandle);
64 
65  void SetETag(const Aws::String& eTag) { m_eTag = eTag; }
66  const Aws::String& GetETag() const { return m_eTag; }
67 
68  Aws::IOStream *GetDownloadPartStream() const { return m_downloadPartStream; }
69  void SetDownloadPartStream(Aws::IOStream *downloadPartStream) { m_downloadPartStream = downloadPartStream; }
70 
71  unsigned char* GetDownloadBuffer() const { return m_downloadBuffer; }
72  void SetDownloadBuffer(unsigned char* downloadBuffer) { m_downloadBuffer = downloadBuffer; }
73 
74  void SetRangeBegin(uint64_t rangeBegin) { m_rangeBegin = rangeBegin; }
75  uint64_t GetRangeBegin() const { return m_rangeBegin; }
76 
77  bool IsLastPart() { return m_lastPart; }
78  void SetLastPart() { m_lastPart = true; }
79 
80  private:
81 
82  int m_partId;
83 
84  Aws::String m_eTag;
85  uint64_t m_currentProgressInBytes;
86  uint64_t m_bestProgressInBytes;
87  uint64_t m_sizeInBytes;
88  uint64_t m_rangeBegin;
89 
90  std::atomic<Aws::IOStream *> m_downloadPartStream;
91  std::atomic<unsigned char*> m_downloadBuffer;
92  bool m_lastPart;
93  };
94 
95  using PartPointer = std::shared_ptr< PartState >;
97 
98  enum class TransferStatus
99  {
100  //this value is only used for directory synchronization
101  EXACT_OBJECT_ALREADY_EXISTS,
102  //Operation is still queued and has not begun processing
103  NOT_STARTED,
104  //Operation is now running
105  IN_PROGRESS,
106  //Operation was canceled. A Canceled operation can still be retried
107  CANCELED,
108  //Operation failed, A failed operaton can still be retried.
109  FAILED,
110  //Operation was successful
111  COMPLETED,
112  //Operation either failed or was canceled and a user deleted the multi-part upload from S3.
113  ABORTED
114  };
115 
116  enum class TransferDirection
117  {
118  UPLOAD,
119  DOWNLOAD
120  };
121 
130  {
131  public:
135  TransferHandle(const Aws::String& bucketName, const Aws::String& keyName, uint64_t totalSize, const Aws::String& targetFilePath = "");
136 
140  TransferHandle(const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& targetFilePath = "");
141 
145  TransferHandle(const Aws::String& bucketName, const Aws::String& keyName, CreateDownloadStreamCallback createDownloadStreamFn, const Aws::String& targetFilePath = "");
146 
150  TransferHandle(const Aws::String& bucketName, const Aws::String& keyName,
151  const uint64_t fileOffset, const uint64_t downloadBytes,
152  CreateDownloadStreamCallback createDownloadStreamFn, const Aws::String& targetFilePath = "");
153 
154 
156 
160  inline bool IsMultipart() const { return m_isMultipart.load(); }
164  inline void SetIsMultipart(bool value) { m_isMultipart.store(value); }
168  inline const Aws::String GetMultiPartId() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_multipartId; }
172  inline void SetMultipartId(const Aws::String& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_multipartId = value; }
180  void ChangePartToCompleted(const PartPointer& partState, const Aws::String &eTag);
188  bool HasPendingParts() const;
192  void AddPendingPart(const PartPointer& partState);
200  bool HasQueuedParts() const;
204  void AddQueuedPart(const PartPointer& partState);
212  bool HasFailedParts() const;
216  void ChangePartToFailed(const PartPointer& partState);
220  void GetAllPartsTransactional(PartStateMap& queuedParts, PartStateMap& pendingParts,
221  PartStateMap& failedParts, PartStateMap& completedParts);
225  bool HasParts() const;
229  bool ShouldContinue() const;
234  void Cancel();
235 
239  void Restart();
249  inline uint64_t GetBytesTransferred() const { return m_bytesTransferred.load(); }
253  void UpdateBytesTransferred(uint64_t amount) { m_bytesTransferred.fetch_add(amount); }
254 
258  inline uint64_t GetBytesOffset() const { return m_offset; }
262  inline uint64_t GetBytesTotalSize() const { return m_bytesTotalSize.load(); }
266  inline void SetBytesTotalSize(uint64_t value) { m_bytesTotalSize.store(value); }
267 
274  inline uint64_t GetBytesAvailableFromStart() const { return m_bytesAvailableFromStart.load(std::memory_order_relaxed); }
275 
279  inline const Aws::String& GetBucketName() const { return m_bucket; }
283  inline const Aws::String& GetKey() const { return m_key; }
288  inline const Aws::String& GetTargetFilePath() const { return m_fileName; }
289 
293  const Aws::String GetVersionId() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_versionId; }
294  void SetVersionId(const Aws::String& versionId) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_versionId = versionId; }
295 
299  inline TransferDirection GetTransferDirection() const { return m_direction; }
303  inline const Aws::String GetContentType() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_contentType; }
307  inline void SetContentType(const Aws::String& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_contentType = value; }
312  inline const Aws::Map<Aws::String, Aws::String> GetMetadata() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_metadata; }
317  inline void SetMetadata(const Aws::Map<Aws::String, Aws::String>& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_metadata = value; }
318 
322  inline void AddMetadataEntry(const Aws::String& key, const Aws::String& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_metadata[key] = value; }
323 
327  inline void SetContext(const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_context = context; }
328 
332  inline std::shared_ptr<const Aws::Client::AsyncCallerContext> GetContext() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_context; }
333 
342 
347  inline const Aws::Client::AWSError<Aws::S3::S3Errors> GetLastError() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_lastError; }
352  inline void SetError(const Aws::Client::AWSError<Aws::S3::S3Errors>& error) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_lastError = error; }
356  void WaitUntilFinished() const;
357 
358  const CreateDownloadStreamCallback& GetCreateDownloadStreamFunction() const { return m_createDownloadStreamFn; }
359 
360  void WritePartToDownloadStream(Aws::IOStream* partStream, uint64_t writeOffset);
361 
363 
365  {
366  bool expected = false;
367  return m_lastPart.compare_exchange_strong(expected, true/*desired*/);
368  }
369 
370  /*
371  * Returns a unique identifier tied to this particular transfer handle.
372  */
374 
375  private:
376 
377  void CleanupDownloadStream();
378 
379  std::atomic<bool> m_isMultipart;
380  Aws::String m_multipartId;
381  TransferDirection m_direction;
382  PartStateMap m_completedParts;
383  PartStateMap m_pendingParts;
384  PartStateMap m_queuedParts;
385  PartStateMap m_failedParts;
386  std::atomic<uint64_t> m_bytesTransferred;
387  std::atomic<bool> m_lastPart;
388  std::atomic<uint64_t> m_bytesTotalSize;
389  std::atomic<uint64_t> m_bytesAvailableFromStart;
390  /* The next part number to watch, that is able to grow m_bytesAvailableFromStart. */
391  uint32_t m_nextPartToWatch;
392  uint64_t m_offset;
393  Aws::String m_bucket;
394  Aws::String m_key;
395  Aws::String m_fileName;
396  Aws::String m_contentType;
397  Aws::String m_versionId;
399  TransferStatus m_status;
401  std::atomic<bool> m_cancel;
402  std::shared_ptr<const Aws::Client::AsyncCallerContext> m_context;
403  const Utils::UUID m_handleId;
404 
405  CreateDownloadStreamCallback m_createDownloadStreamFn;
406  Aws::IOStream* m_downloadStream;
407  /* in case cutomer stream is not based off 0 */
408  uint64_t m_downloadStreamBaseOffset;
409 
410  mutable std::mutex m_downloadStreamLock;
411  mutable std::mutex m_partsLock;
412  mutable std::mutex m_statusLock;
413  mutable std::condition_variable m_waitUntilFinishedSignal;
414  mutable std::mutex m_getterSetterLock;
415  };
416 
418  }
419 }
Aws::Transfer::TransferDirection
TransferDirection
Definition: TransferHandle.h:117
Aws::Transfer::PartState::GetETag
const Aws::String & GetETag() const
Definition: TransferHandle.h:66
Aws::IOStream
std::basic_iostream< char, std::char_traits< char > > IOStream
Definition: AWSStreamFwd.h:22
Aws::Transfer::TransferHandle::GetKey
const Aws::String & GetKey() const
Definition: TransferHandle.h:283
Aws::Transfer::TransferHandle::AddQueuedPart
void AddQueuedPart(const PartPointer &partState)
Aws::Transfer::PartState::GetRangeBegin
uint64_t GetRangeBegin() const
Definition: TransferHandle.h:75
Aws::Transfer::PartPointer
std::shared_ptr< PartState > PartPointer
Definition: TransferHandle.h:95
Aws::Transfer::TransferHandle::GetMetadata
const Aws::Map< Aws::String, Aws::String > GetMetadata() const
Definition: TransferHandle.h:312
Aws::Transfer::TransferHandle::TransferHandle
TransferHandle(const Aws::String &bucketName, const Aws::String &keyName, CreateDownloadStreamCallback createDownloadStreamFn, const Aws::String &targetFilePath="")
AWSError.h
Aws::Transfer::PartStateMap
Aws::Map< int, PartPointer > PartStateMap
Definition: TransferHandle.h:96
Aws::Transfer::PartState::GetBestProgressInBytes
uint64_t GetBestProgressInBytes() const
Definition: TransferHandle.h:55
Aws::Transfer::TransferHandle::GetAllPartsTransactional
void GetAllPartsTransactional(PartStateMap &queuedParts, PartStateMap &pendingParts, PartStateMap &failedParts, PartStateMap &completedParts)
Aws::Transfer::TransferHandle::SetVersionId
void SetVersionId(const Aws::String &versionId)
Definition: TransferHandle.h:294
Aws::Transfer::TransferHandle::GetStatus
TransferStatus GetStatus() const
Aws::Transfer::PartState::SetLastPart
void SetLastPart()
Definition: TransferHandle.h:78
Aws::Transfer::TransferHandle::SetMetadata
void SetMetadata(const Aws::Map< Aws::String, Aws::String > &value)
Definition: TransferHandle.h:317
Aws::Transfer::TransferHandle::GetContext
std::shared_ptr< const Aws::Client::AsyncCallerContext > GetContext() const
Definition: TransferHandle.h:332
Aws::Transfer::DownloadConfiguration::DownloadConfiguration
DownloadConfiguration()
Definition: TransferHandle.h:38
Aws::Transfer::TransferHandle::GetId
Aws::String GetId() const
Aws::Transfer::TransferHandle::~TransferHandle
~TransferHandle()
Aws::Transfer::TransferHandle::SetContentType
void SetContentType(const Aws::String &value)
Definition: TransferHandle.h:307
Aws::Transfer::TransferHandle::GetMultiPartId
const Aws::String GetMultiPartId() const
Definition: TransferHandle.h:168
Aws::Transfer::TransferHandle::ChangePartToCompleted
void ChangePartToCompleted(const PartPointer &partState, const Aws::String &eTag)
AsyncCallerContext.h
Aws::Transfer::TransferHandle::HasFailedParts
bool HasFailedParts() const
Aws::Transfer::PartState::SetSizeInBytes
void SetSizeInBytes(uint64_t sizeInBytes)
Definition: TransferHandle.h:59
Aws::Transfer::PartState::GetSizeInBytes
uint64_t GetSizeInBytes() const
Definition: TransferHandle.h:58
Aws::Transfer::TransferHandle::GetTargetFilePath
const Aws::String & GetTargetFilePath() const
Definition: TransferHandle.h:288
Aws::Transfer::TransferHandle::SetIsMultipart
void SetIsMultipart(bool value)
Definition: TransferHandle.h:164
Aws::Transfer::TransferHandle::HasQueuedParts
bool HasQueuedParts() const
Aws::Transfer::TransferHandle::UpdateBytesTransferred
void UpdateBytesTransferred(uint64_t amount)
Definition: TransferHandle.h:253
Aws::Transfer::TransferHandle::GetBytesTotalSize
uint64_t GetBytesTotalSize() const
Definition: TransferHandle.h:262
Aws::Transfer::TransferHandle::SetBytesTotalSize
void SetBytesTotalSize(uint64_t value)
Definition: TransferHandle.h:266
S3Errors.h
Aws::Transfer::TransferHandle::GetLastError
const Aws::Client::AWSError< Aws::S3::S3Errors > GetLastError() const
Definition: TransferHandle.h:347
Aws::OStream
std::basic_ostream< char, std::char_traits< char > > OStream
Definition: AWSStreamFwd.h:21
Aws::Transfer::TransferHandle::WritePartToDownloadStream
void WritePartToDownloadStream(Aws::IOStream *partStream, uint64_t writeOffset)
Aws::Transfer::TransferHandle::GetVersionId
const Aws::String GetVersionId() const
Definition: TransferHandle.h:293
Aws::Transfer::TransferHandle::LockForCompletion
bool LockForCompletion()
Definition: TransferHandle.h:364
Aws::Transfer::operator<<
AWS_TRANSFER_API Aws::OStream & operator<<(Aws::OStream &s, TransferStatus status)
Aws::Transfer::PartState::OnDataTransferred
void OnDataTransferred(uint64_t amount, const std::shared_ptr< TransferHandle > &transferHandle)
Aws::Transfer::TransferHandle::TransferHandle
TransferHandle(const Aws::String &bucketName, const Aws::String &keyName, const Aws::String &targetFilePath="")
Aws::Transfer::TransferHandle::GetContentType
const Aws::String GetContentType() const
Definition: TransferHandle.h:303
Aws::Transfer::TransferHandle::GetCreateDownloadStreamFunction
const CreateDownloadStreamCallback & GetCreateDownloadStreamFunction() const
Definition: TransferHandle.h:358
Aws::Transfer::TransferHandle::UpdateStatus
void UpdateStatus(TransferStatus value)
Aws::Transfer::CreateDownloadStreamCallback
std::function< Aws::IOStream *(void)> CreateDownloadStreamCallback
Definition: TransferHandle.h:30
Aws::Transfer::DownloadConfiguration
Definition: TransferHandle.h:37
Aws::Transfer::TransferHandle::AddPendingPart
void AddPendingPart(const PartPointer &partState)
Aws::Transfer::TransferStatus::EXACT_OBJECT_ALREADY_EXISTS
@ EXACT_OBJECT_ALREADY_EXISTS
Aws::Transfer::TransferHandle::GetQueuedParts
PartStateMap GetQueuedParts() const
Aws::Transfer::PartState::GetDownloadBuffer
unsigned char * GetDownloadBuffer() const
Definition: TransferHandle.h:71
Aws::Transfer::TransferHandle
Definition: TransferHandle.h:130
Aws::Transfer::TransferHandle::GetFailedParts
PartStateMap GetFailedParts() const
Aws::Transfer::TransferHandle::HasPendingParts
bool HasPendingParts() const
Aws::Transfer::TransferHandle::SetContext
void SetContext(const std::shared_ptr< const Aws::Client::AsyncCallerContext > &context)
Definition: TransferHandle.h:327
AWSString.h
Aws::Transfer::TransferHandle::TransferHandle
TransferHandle(const Aws::String &bucketName, const Aws::String &keyName, uint64_t totalSize, const Aws::String &targetFilePath="")
Aws::Transfer::TransferHandle::GetTransferDirection
TransferDirection GetTransferDirection() const
Definition: TransferHandle.h:299
Aws::Transfer::TransferHandle::GetCompletedParts
PartStateMap GetCompletedParts() const
Aws::Transfer::TransferHandle::AddMetadataEntry
void AddMetadataEntry(const Aws::String &key, const Aws::String &value)
Definition: TransferHandle.h:322
Aws::Client::AWSError< Aws::S3::S3Errors >
Aws::Transfer::TransferHandle::Restart
void Restart()
Aws::Transfer::TransferHandle::SetMultipartId
void SetMultipartId(const Aws::String &value)
Definition: TransferHandle.h:172
Aws::Transfer::TransferHandle::GetPendingParts
PartStateMap GetPendingParts() const
Aws::Transfer::PartState::PartState
PartState()
Aws::Transfer::TransferHandle::GetBucketName
const Aws::String & GetBucketName() const
Definition: TransferHandle.h:279
Aws::Transfer::PartState::SetDownloadPartStream
void SetDownloadPartStream(Aws::IOStream *downloadPartStream)
Definition: TransferHandle.h:69
Aws::Transfer::TransferHandle::WaitUntilFinished
void WaitUntilFinished() const
Aws::Transfer::PartState::GetDownloadPartStream
Aws::IOStream * GetDownloadPartStream() const
Definition: TransferHandle.h:68
Aws::Transfer::PartState::SetBestProgressInBytes
void SetBestProgressInBytes(uint64_t progressInBytes)
Definition: TransferHandle.h:56
Aws::Transfer::PartState::GetPartId
int GetPartId() const
Definition: TransferHandle.h:53
Aws::Transfer::PartState::IsLastPart
bool IsLastPart()
Definition: TransferHandle.h:77
Aws::Transfer::PartState
Definition: TransferHandle.h:48
Aws::Transfer::TransferHandle::GetBytesTransferred
uint64_t GetBytesTransferred() const
Definition: TransferHandle.h:249
Aws::Transfer::TransferHandle::HasParts
bool HasParts() const
Aws::Transfer::TransferHandle::SetError
void SetError(const Aws::Client::AWSError< Aws::S3::S3Errors > &error)
Definition: TransferHandle.h:352
Aws
Definition: AccessManagementClient.h:15
Aws::Transfer::TransferHandle::ChangePartToFailed
void ChangePartToFailed(const PartPointer &partState)
Aws::Transfer::PartState::SetDownloadBuffer
void SetDownloadBuffer(unsigned char *downloadBuffer)
Definition: TransferHandle.h:72
Aws::Transfer::DownloadConfiguration::versionId
Aws::String versionId
Definition: TransferHandle.h:42
Aws::Transfer::PartState::SetRangeBegin
void SetRangeBegin(uint64_t rangeBegin)
Definition: TransferHandle.h:74
Aws::Transfer::TransferHandle::IsMultipart
bool IsMultipart() const
Definition: TransferHandle.h:160
Aws::String
std::basic_string< char, std::char_traits< char >, Aws::Allocator< char > > String
Definition: AWSString.h:97
Aws::Utils::UUID
Definition: UUID.h:22
Aws::Transfer::PartState::Reset
void Reset()
Aws::Transfer::TransferHandle::GetBytesOffset
uint64_t GetBytesOffset() const
Definition: TransferHandle.h:258
Aws::Transfer::PartState::PartState
PartState(int partId, uint64_t bestProgressInBytes, uint64_t sizeInBytes, bool lastPart=false)
Aws::Transfer::CLASS_TAG
static const char CLASS_TAG[]
Definition: TransferHandle.h:34
Aws::Transfer::TransferHandle::GetBytesAvailableFromStart
uint64_t GetBytesAvailableFromStart() const
Definition: TransferHandle.h:274
UUID.h
Aws::Transfer::TransferHandle::Cancel
void Cancel()
AWSMap.h
AWSSet.h
Aws::Transfer::TransferDirection::UPLOAD
@ UPLOAD
Aws::Transfer::PartState::SetETag
void SetETag(const Aws::String &eTag)
Definition: TransferHandle.h:65
Aws::Map
std::map< K, V, std::less< K >, Aws::Allocator< std::pair< const K, V > > > Map
Definition: AWSMap.h:19
Aws::Transfer::TransferHandle::ApplyDownloadConfiguration
void ApplyDownloadConfiguration(const DownloadConfiguration &downloadConfig)
Aws::Transfer::TransferHandle::ShouldContinue
bool ShouldContinue() const
Transfer_EXPORTS.h
Aws::Transfer::TransferStatus
TransferStatus
Definition: TransferHandle.h:99
Aws::Transfer::TransferHandle::TransferHandle
TransferHandle(const Aws::String &bucketName, const Aws::String &keyName, const uint64_t fileOffset, const uint64_t downloadBytes, CreateDownloadStreamCallback createDownloadStreamFn, const Aws::String &targetFilePath="")
AWS_TRANSFER_API
#define AWS_TRANSFER_API
Definition: Transfer_EXPORTS.h:28