From 184a20540beb47963afbca64bd9a3e96dbb7b020 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Nov 2015 11:14:03 -0400 Subject: Fix failure to build with aws-0.13.0 and finish nearline support. * Fix failure to build with aws-0.13.0. * When built with aws-0.13.0, the S3 special remote can be used to create google nearline buckets, by setting storageclass=NEARLINE. --- Remote/S3.hs | 10 ++++++++++ debian/changelog | 3 +++ ...fault_storage_classes_with_Google_Cloud_Storage.mdwn | 2 ++ .../comment_5_ab6cf3c57d9f93c24c19df81582f38d5._comment | 11 +++++++++++ doc/special_remotes/S3.mdwn | 9 +++++++-- .../comment_5_865f27a393f24fd5723c895d4d70dde3._comment | 17 +++++++++++++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage/comment_5_ab6cf3c57d9f93c24c19df81582f38d5._comment create mode 100644 doc/tips/using_Google_Cloud_Storage/comment_5_865f27a393f24fd5723c895d4d70dde3._comment diff --git a/Remote/S3.hs b/Remote/S3.hs index fa88b84bc..fb772825c 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -330,10 +330,20 @@ genBucket c u = do (bucket info) (acl info) locconstraint +#if MIN_VERSION_aws(0,13,0) + storageclass +#endif writeUUIDFile c u info h locconstraint = mkLocationConstraint $ T.pack datacenter datacenter = fromJust $ M.lookup "datacenter" c +#if MIN_VERSION_aws(0,13,0) + -- "NEARLINE" as a storage class when creating a bucket is a + -- nonstandard extension of Google Cloud Storage. + storageclass = case getStorageClass c of + sc@(S3.OtherStorageClass "NEARLINE") -> Just sc + _ -> Nothing +#endif {- Writes the UUID to an annex-uuid file within the bucket. - diff --git a/debian/changelog b/debian/changelog index 8fd9c71e5..9bb8a2522 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,9 @@ git-annex (5.20151020) UNRELEASED; urgency=medium * Catch up with current git behavior when both repo and repo.git exist; it seems it now prefers repo in this case, although historically it may have preferred repo.git. + * Fix failure to build with aws-0.13.0. + * When built with aws-0.13.0, the S3 special remote can be used to create + google nearline buckets, by setting storageclass=NEARLINE. -- Joey Hess Mon, 19 Oct 2015 17:00:21 -0400 diff --git a/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage.mdwn b/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage.mdwn index 828dc72c0..1c11e45ba 100644 --- a/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage.mdwn +++ b/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage.mdwn @@ -90,3 +90,5 @@ initremote cloud (checking bucket...) [2015-05-31 17:38:21 EDT] String to sign: [2015-05-31 17:38:22 EDT] Response metadata: S3: request ID=, x-amz-id-2= git-annex: S3Error {s3StatusCode = Status {statusCode = 400, statusMessage = "Bad Request"}, s3ErrorCode = "InvalidArgument", s3ErrorMessage = "Invalid argument.", s3ErrorResource = Nothing, s3ErrorHostId = Nothing, s3ErrorAccessKeyId = Nothing, s3ErrorStringToSign = Nothing} """]] + +> [[done]], see comments --[[Joey]] diff --git a/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage/comment_5_ab6cf3c57d9f93c24c19df81582f38d5._comment b/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage/comment_5_ab6cf3c57d9f93c24c19df81582f38d5._comment new file mode 100644 index 000000000..4e87d7a5f --- /dev/null +++ b/doc/bugs/Support_non-default_storage_classes_with_Google_Cloud_Storage/comment_5_ab6cf3c57d9f93c24c19df81582f38d5._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-11-02T14:55:54Z" + content=""" +aws-0.13.0 was released with the ncessary support. git-annex will now pass +the NEARLINE storage class when creating a bucket. + +Note that most builds of git-annex are not yet made with that version of +aws, but it will trickle out to the build systems with time. +"""]] diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn index f4d1c552e..f6c01c0d0 100644 --- a/doc/special_remotes/S3.mdwn +++ b/doc/special_remotes/S3.mdwn @@ -43,9 +43,14 @@ the S3 remote. When using Amazon S3, if you have configured git-annex to preserve multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY" - to save money. Or, if the remote will be used for backup or archival, + to save money. + + Or, if the remote will be used for backup or archival, and so its files are Infrequently Accessed, "STANDARD_IA" is also a - good choice to save money. + good choice to save money. (Requires a git-annex built with aws-0.13.0) + + When using Google Cloud Storage, to make a nearline bucket, set this to + "NEARLINE". (Requires a git-annex built with aws-0.13.0) Note that changing the storage class of an existing S3 remote will affect new objects sent to the remote, but not objects already diff --git a/doc/tips/using_Google_Cloud_Storage/comment_5_865f27a393f24fd5723c895d4d70dde3._comment b/doc/tips/using_Google_Cloud_Storage/comment_5_865f27a393f24fd5723c895d4d70dde3._comment new file mode 100644 index 000000000..6d1a895a2 --- /dev/null +++ b/doc/tips/using_Google_Cloud_Storage/comment_5_865f27a393f24fd5723c895d4d70dde3._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-11-02T14:59:08Z" + content=""" +Nice job bgilbert on the external special remote. + +git-annex's S3 special remote also supports nearline now: + + git annex initremote cloud type=S3 encryption=none host=storage.googleapis.com port=80 storageclass=NEARLINE + +For the nearline support to work, git-annex has to be built with version 0.13.0 of +the haskell aws library. + +But, it seems like bgilbert's external special remote is easier to set up, +due to not needing to configure Google Cloud Storage for interoperable access. +"""]] -- cgit v1.2.3