object.go 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121
  1. package tos
  2. import (
  3. "bytes"
  4. "context"
  5. "errors"
  6. "fmt"
  7. "hash"
  8. "io"
  9. "io/ioutil"
  10. "net/http"
  11. "os"
  12. "path/filepath"
  13. "strconv"
  14. )
  15. type Bucket struct {
  16. name string
  17. client *Client
  18. baseClient *baseClient
  19. }
  20. // GetObject get data and metadata of an object
  21. // objectKey: the name of object
  22. // options: WithVersionID which version of this object
  23. // WithRange the range of content,
  24. // WithIfModifiedSince return if the object modified after the given date, otherwise return status code 304
  25. // WithIfUnmodifiedSince, WithIfMatch, WithIfNoneMatch set If-Unmodified-Since, If-Match and If-None-Match
  26. //
  27. // Deprecated: use GetObject of ClientV2 instead
  28. func (bkt *Bucket) GetObject(ctx context.Context, objectKey string, options ...Option) (*GetObjectOutput, error) {
  29. if err := isValidKey(objectKey); err != nil {
  30. return nil, err
  31. }
  32. rb := bkt.client.newBuilder(bkt.name, objectKey, options...)
  33. res, err := rb.WithRetry(nil, StatusCodeClassifier{}).Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(expectedCode(rb)))
  34. if err != nil {
  35. return nil, err
  36. }
  37. output := GetObjectOutput{
  38. RequestInfo: res.RequestInfo(),
  39. ContentRange: rb.Header.Get(HeaderContentRange),
  40. Content: res.Body,
  41. }
  42. output.ObjectMeta.fromResponse(res)
  43. return &output, nil
  44. }
  45. func (cli *ClientV2) copyToFile(fileName string, reader io.Reader) error {
  46. fd, err := os.OpenFile(filepath.Clean(fileName), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultFilePerm)
  47. if err != nil {
  48. return err
  49. }
  50. defer fd.Close()
  51. _, err = io.Copy(fd, reader)
  52. if err != nil {
  53. return err
  54. }
  55. return nil
  56. }
  57. // GetObjectToFile get object and write it to file
  58. func (cli *ClientV2) GetObjectToFile(ctx context.Context, input *GetObjectToFileInput) (*GetObjectToFileOutput, error) {
  59. err := checkAndCreateDir(input.FilePath)
  60. if err != nil {
  61. return nil, InvalidFilePath.withCause(err)
  62. }
  63. tempFilePath := input.FilePath + TempFileSuffix
  64. get, err := cli.GetObjectV2(ctx, &input.GetObjectV2Input)
  65. if err != nil {
  66. return nil, err
  67. }
  68. defer get.Content.Close()
  69. err = cli.copyToFile(tempFilePath, get.Content)
  70. if err != nil {
  71. return nil, newTosClientError("GetObject to File error", err)
  72. }
  73. err = os.Rename(tempFilePath, input.FilePath)
  74. if err != nil {
  75. return nil, err
  76. }
  77. return &GetObjectToFileOutput{get.GetObjectBasicOutput}, nil
  78. }
  79. // GetObjectV2 get data and metadata of an object
  80. func (cli *ClientV2) GetObjectV2(ctx context.Context, input *GetObjectV2Input) (*GetObjectV2Output, error) {
  81. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  82. return nil, err
  83. }
  84. rb := cli.newBuilder(input.Bucket, input.Key).
  85. WithParams(*input).WithRetry(nil, StatusCodeClassifier{})
  86. if input.Range != "" {
  87. rb.WithHeader(HeaderRange, input.Range)
  88. } else if input.RangeEnd != 0 || input.RangeStart != 0 {
  89. if input.RangeEnd < input.RangeStart {
  90. return nil, errors.New("tos: invalid range")
  91. }
  92. // set rb.Range will change expected code
  93. rb.Range = &Range{Start: input.RangeStart, End: input.RangeEnd}
  94. rb.WithHeader(HeaderRange, rb.Range.String())
  95. }
  96. res, err := rb.Request(ctx, http.MethodGet, nil, cli.roundTripper(expectedCode(rb)))
  97. if err != nil {
  98. return nil, err
  99. }
  100. basic := GetObjectBasicOutput{
  101. RequestInfo: res.RequestInfo(),
  102. ContentRange: res.Header.Get(HeaderContentRange),
  103. }
  104. basic.ObjectMetaV2.fromResponseV2(res)
  105. var serverCrc uint64
  106. var checker hash.Hash64
  107. // 200 为完整请求
  108. if res.StatusCode == http.StatusOK && cli.enableCRC {
  109. serverCrc = basic.HashCrc64ecma
  110. checker = NewCRC(DefaultCrcTable(), 0)
  111. }
  112. output := GetObjectV2Output{
  113. GetObjectBasicOutput: basic,
  114. Content: wrapReader(res.Body, res.ContentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker, serverCrc: serverCrc}),
  115. }
  116. return &output, nil
  117. }
  118. // HeadObject get metadata of an object
  119. // objectKey: the name of object
  120. // options: WithVersionID which version of this object
  121. // WithRange the range of content,
  122. // WithIfModifiedSince return if the object modified after the given date, otherwise return status code 304
  123. // WithIfUnmodifiedSince, WithIfMatch, WithIfNoneMatch set If-Unmodified-Since, If-Match and If-None-Match
  124. //
  125. // Deprecated: use HeadObject of ClientV2 instead
  126. func (bkt *Bucket) HeadObject(ctx context.Context, objectKey string, options ...Option) (*HeadObjectOutput, error) {
  127. if err := isValidKey(objectKey); err != nil {
  128. return nil, err
  129. }
  130. rb := bkt.client.newBuilder(bkt.name, objectKey, options...)
  131. res, err := rb.WithRetry(nil, StatusCodeClassifier{}).Request(ctx, http.MethodHead, nil, bkt.client.roundTripper(expectedCode(rb)))
  132. if err != nil {
  133. return nil, err
  134. }
  135. defer res.Close()
  136. output := HeadObjectOutput{
  137. RequestInfo: res.RequestInfo(),
  138. ContentRange: rb.Header.Get(HeaderContentRange),
  139. }
  140. output.ObjectMeta.fromResponse(res)
  141. return &output, nil
  142. }
  143. // HeadObjectV2 get metadata of an object
  144. func (cli *ClientV2) HeadObjectV2(ctx context.Context, input *HeadObjectV2Input) (*HeadObjectV2Output, error) {
  145. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  146. return nil, err
  147. }
  148. rb := cli.newBuilder(input.Bucket, input.Key).
  149. WithParams(*input).
  150. WithRetry(nil, StatusCodeClassifier{})
  151. res, err := rb.Request(ctx, http.MethodHead, nil, cli.roundTripper(expectedCode(rb)))
  152. if err != nil {
  153. return nil, err
  154. }
  155. defer res.Close()
  156. output := HeadObjectV2Output{
  157. RequestInfo: res.RequestInfo(),
  158. }
  159. output.ObjectMetaV2.fromResponseV2(res)
  160. return &output, nil
  161. }
  162. func expectedCode(rb *requestBuilder) int {
  163. okCode := http.StatusOK
  164. if rb.Header.Get(HeaderRange) != "" || rb.Query.Get(QueryPartNumber) != "" {
  165. okCode = http.StatusPartialContent
  166. }
  167. return okCode
  168. }
  169. // DeleteObject delete an object
  170. // objectKey: the name of object
  171. // options: WithVersionID which version of this object will be deleted
  172. //
  173. // Deprecated: use DeleteObject of ClientV2 instead
  174. func (bkt *Bucket) DeleteObject(ctx context.Context, objectKey string, options ...Option) (*DeleteObjectOutput, error) {
  175. if err := isValidKey(objectKey); err != nil {
  176. return nil, err
  177. }
  178. res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).WithRetry(nil, StatusCodeClassifier{}).
  179. Request(ctx, http.MethodDelete, nil, bkt.client.roundTripper(http.StatusNoContent))
  180. if err != nil {
  181. return nil, err
  182. }
  183. defer res.Close()
  184. deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker))
  185. return &DeleteObjectOutput{
  186. RequestInfo: res.RequestInfo(),
  187. DeleteMarker: deleteMarker,
  188. VersionID: res.Header.Get(HeaderVersionID),
  189. }, nil
  190. }
  191. // DeleteObjectV2 delete an object
  192. func (cli *ClientV2) DeleteObjectV2(ctx context.Context, input *DeleteObjectV2Input) (*DeleteObjectV2Output, error) {
  193. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  194. return nil, err
  195. }
  196. res, err := cli.newBuilder(input.Bucket, input.Key).
  197. WithParams(*input).
  198. WithRetry(nil, StatusCodeClassifier{}).
  199. Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
  200. if err != nil {
  201. return nil, err
  202. }
  203. defer res.Close()
  204. deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker))
  205. return &DeleteObjectV2Output{
  206. DeleteObjectOutput{
  207. RequestInfo: res.RequestInfo(),
  208. DeleteMarker: deleteMarker,
  209. VersionID: res.Header.Get(HeaderVersionID)}}, nil
  210. }
  211. // DeleteMultiObjects delete multi-objects
  212. // input: the objects will be deleted
  213. //
  214. // Deprecated: use DeleteMultiObjects of ClientV2 instead
  215. func (bkt *Bucket) DeleteMultiObjects(ctx context.Context, input *DeleteMultiObjectsInput, options ...Option) (*DeleteMultiObjectsOutput, error) {
  216. for _, object := range input.Objects {
  217. if err := isValidKey(object.Key); err != nil {
  218. return nil, err
  219. }
  220. }
  221. in, contentMD5, err := marshalInput("DeleteMultiObjectsInput", deleteMultiObjectsInput{
  222. Objects: input.Objects,
  223. Quiet: input.Quiet,
  224. })
  225. if err != nil {
  226. return nil, err
  227. }
  228. res, err := bkt.client.newBuilder(bkt.name, "", options...).
  229. WithHeader(HeaderContentMD5, contentMD5).
  230. WithQuery("delete", "").
  231. WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
  232. Request(ctx, http.MethodPost, bytes.NewReader(in), bkt.client.roundTripper(http.StatusOK))
  233. if err != nil {
  234. return nil, err
  235. }
  236. defer res.Close()
  237. output := DeleteMultiObjectsOutput{RequestInfo: res.RequestInfo()}
  238. if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
  239. return nil, err
  240. }
  241. return &output, nil
  242. }
  243. // DeleteMultiObjects delete multi-objects
  244. func (cli *ClientV2) DeleteMultiObjects(ctx context.Context, input *DeleteMultiObjectsInput) (*DeleteMultiObjectsOutput, error) {
  245. if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
  246. return nil, err
  247. }
  248. if len(input.Objects) == 0 {
  249. return nil, InvlidDeleteMultiObjectsLength
  250. }
  251. for _, object := range input.Objects {
  252. if err := isValidKey(object.Key); err != nil {
  253. return nil, err
  254. }
  255. }
  256. in, contentMD5, err := marshalInput("DeleteMultiObjectsInput", deleteMultiObjectsInput{
  257. Objects: input.Objects,
  258. Quiet: input.Quiet,
  259. })
  260. if err != nil {
  261. return nil, err
  262. }
  263. // POST method, don't retry
  264. res, err := cli.newBuilder(input.Bucket, "").
  265. WithQuery("delete", "").
  266. WithHeader(HeaderContentMD5, contentMD5).
  267. WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
  268. Request(ctx, http.MethodPost, bytes.NewReader(in), cli.roundTripper(http.StatusOK))
  269. if err != nil {
  270. return nil, err
  271. }
  272. defer res.Close()
  273. output := DeleteMultiObjectsOutput{RequestInfo: res.RequestInfo()}
  274. if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
  275. return nil, err
  276. }
  277. return &output, nil
  278. }
  279. // PutObject put an object
  280. // objectKey: the name of object
  281. // content: the content of object
  282. // options: WithContentType set Content-Type,
  283. // WithContentDisposition set Content-Disposition,
  284. // WithContentLanguage set Content-Language,
  285. // WithContentEncoding set Content-Encoding,
  286. // WithCacheControl set Cache-Control,
  287. // WithExpires set Expires,
  288. // WithMeta set meta header(s),
  289. // WithContentSHA256 set Content-Sha256,
  290. // WithContentMD5 set Content-MD5
  291. // WithExpires set Expires,
  292. // WithServerSideEncryptionCustomer set server side encryption options
  293. // WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl
  294. //
  295. // NOTICE: only content with a known length is supported now,
  296. // e.g, bytes.Buffer, bytes.Reader, strings.Reader, os.File, io.LimitedReader, net.Buffers.
  297. // if the parameter content(an io.Reader) is not one of these,
  298. // please use io.LimitReader(reader, length) to wrap this reader or use the WithContentLength option.
  299. //
  300. // Deprecated: use PutObjectV2 of ClientV2 instead
  301. func (bkt *Bucket) PutObject(ctx context.Context, objectKey string, content io.Reader, options ...Option) (*PutObjectOutput, error) {
  302. if err := isValidKey(objectKey); err != nil {
  303. return nil, err
  304. }
  305. var (
  306. onRetry func(req *Request) error = nil
  307. classifier classifier
  308. )
  309. classifier = NoRetryClassifier{}
  310. if seeker, ok := content.(io.Seeker); ok {
  311. start, err := seeker.Seek(0, io.SeekCurrent)
  312. if err == nil {
  313. onRetry = func(req *Request) error {
  314. // PutObject/UploadPart can be treated as an idempotent semantics if the request message body
  315. // supports a reset operation. e.g. the request message body is a string,
  316. // a local file handle, binary data in memory
  317. if seeker, ok := req.Content.(io.Seeker); ok {
  318. _, err := seeker.Seek(start, io.SeekStart)
  319. if err != nil {
  320. return err
  321. }
  322. } else {
  323. return newTosClientError("Io Reader not support retry", nil)
  324. }
  325. return nil
  326. }
  327. classifier = StatusCodeClassifier{}
  328. }
  329. }
  330. res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).
  331. WithRetry(onRetry, classifier).
  332. Request(ctx, http.MethodPut, content, bkt.client.roundTripper(http.StatusOK))
  333. if err != nil {
  334. return nil, err
  335. }
  336. defer res.Close()
  337. return &PutObjectOutput{
  338. RequestInfo: res.RequestInfo(),
  339. ETag: res.Header.Get(HeaderETag),
  340. VersionID: res.Header.Get(HeaderVersionID),
  341. SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
  342. SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
  343. }, nil
  344. }
  345. func skipEscape(i byte) bool {
  346. return (i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') ||
  347. i == '-' ||
  348. i == '.' ||
  349. i == '_' ||
  350. i == '~'
  351. }
  352. func escapeHeader(s string) string {
  353. var buf bytes.Buffer
  354. for i := 0; i < len(s); i++ {
  355. c := s[i]
  356. if skipEscape(c) {
  357. buf.WriteByte(c)
  358. } else {
  359. fmt.Fprintf(&buf, "%%%02X", c)
  360. }
  361. }
  362. return buf.String()
  363. }
  364. func existChinese(s string) bool {
  365. r := []rune(s)
  366. for i := 0; i < len(r); i++ {
  367. if r[i] >= 0x4E00 && r[i] <= 0x9FA5 {
  368. return true
  369. }
  370. }
  371. return false
  372. }
  373. // url-encode Chinese characters only
  374. func headerEncode(s string) string {
  375. return escapeHeader(s)
  376. }
  377. func checkCrc64(res *Response, checker hash.Hash64) error {
  378. if res.Header.Get(HeaderHashCrc64ecma) == "" || checker == nil {
  379. return nil
  380. }
  381. crc64, err := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
  382. if err != nil {
  383. return &TosServerError{
  384. TosError: TosError{"tos: server returned invalid crc"},
  385. RequestInfo: res.RequestInfo(),
  386. }
  387. }
  388. if checker.Sum64() != crc64 {
  389. return &TosServerError{
  390. TosError: TosError{Message: fmt.Sprintf("tos: crc64 check failed, expected:%d, in fact:%d", crc64, checker.Sum64())},
  391. RequestInfo: res.RequestInfo(),
  392. }
  393. }
  394. return nil
  395. }
  396. type crcChecker struct {
  397. checker hash.Hash64
  398. serverCrc uint64
  399. }
  400. type nopCloser struct {
  401. base io.Reader
  402. }
  403. func wrapCloser(reader io.Reader) io.ReadCloser {
  404. return &nopCloser{base: reader}
  405. }
  406. func (n2 nopCloser) Seek(offset int64, whence int) (int64, error) {
  407. seeker, ok := n2.base.(io.Seeker)
  408. if !ok {
  409. return 0, NotSupportSeek
  410. }
  411. return seeker.Seek(offset, whence)
  412. }
  413. func (n2 nopCloser) Read(p []byte) (n int, err error) {
  414. return n2.base.Read(p)
  415. }
  416. func (n2 nopCloser) Close() error {
  417. return nil
  418. }
  419. // wrapReader wrap reader with some extension function.
  420. // If reader can be interpreted as io.ReadCloser, use itself as base ReadCloser, else wrap it a NopCloser.
  421. func wrapReader(reader io.Reader, totalBytes int64, listener DataTransferListener, limiter RateLimiter, crcChecker *crcChecker) io.ReadCloser {
  422. var wrapped io.ReadCloser
  423. // get base ReadCloser
  424. if rc, ok := reader.(io.ReadCloser); ok {
  425. wrapped = rc
  426. } else {
  427. wrapped = wrapCloser(reader)
  428. }
  429. // wrap with listener
  430. if listener != nil {
  431. wrapped = &readCloserWithListener{
  432. listener: listener,
  433. base: wrapped,
  434. consumed: 0,
  435. total: totalBytes,
  436. }
  437. }
  438. // wrap with limiter
  439. if limiter != nil {
  440. wrapped = &ReadCloserWithLimiter{
  441. limiter: limiter,
  442. base: wrapped,
  443. }
  444. }
  445. // wrap with crc64 checker
  446. if crcChecker != nil && crcChecker.checker != nil {
  447. wrapped = &readCloserWithCRC{
  448. serverCrc: crcChecker.serverCrc,
  449. checker: crcChecker.checker,
  450. base: wrapped,
  451. }
  452. }
  453. return wrapped
  454. }
  455. // PutObjectV2 put an object
  456. func (cli *ClientV2) PutObjectV2(ctx context.Context, input *PutObjectV2Input) (*PutObjectV2Output, error) {
  457. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  458. return nil, err
  459. }
  460. if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil {
  461. return nil, err
  462. }
  463. if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil {
  464. return nil, err
  465. }
  466. if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil {
  467. return nil, err
  468. }
  469. var (
  470. checker hash.Hash64
  471. content = input.Content
  472. contentLength = input.ContentLength
  473. )
  474. if cli.enableCRC {
  475. checker = NewCRC(DefaultCrcTable(), 0)
  476. }
  477. if contentLength <= 0 {
  478. contentLength = tryResolveLength(content)
  479. }
  480. var (
  481. onRetry func(req *Request) error = nil
  482. classifier classifier
  483. )
  484. if content != nil {
  485. content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker})
  486. }
  487. classifier = NoRetryClassifier{}
  488. if seeker, ok := content.(io.Seeker); ok {
  489. start, err := seeker.Seek(0, io.SeekCurrent)
  490. if err == nil {
  491. onRetry = func(req *Request) error {
  492. // PutObject/UploadPart can be treated as an idempotent semantics if the request message body
  493. // supports a reset operation. e.g. the request message body is a string,
  494. // a local file handle, binary data in memory
  495. if seeker, ok := req.Content.(io.Seeker); ok {
  496. _, err := seeker.Seek(start, io.SeekStart)
  497. if err != nil {
  498. return err
  499. }
  500. } else {
  501. return newTosClientError("Io Reader not support retry", nil)
  502. }
  503. return nil
  504. }
  505. classifier = StatusCodeClassifier{}
  506. }
  507. }
  508. rb := cli.newBuilder(input.Bucket, input.Key).
  509. WithContentLength(contentLength).
  510. WithParams(*input).
  511. WithRetry(onRetry, classifier)
  512. res, err := rb.Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK))
  513. if err != nil {
  514. return nil, err
  515. }
  516. defer res.Close()
  517. if err = checkCrc64(res, checker); err != nil {
  518. return nil, err
  519. }
  520. crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
  521. callbackResult := ""
  522. if input.Callback != "" && res.Body != nil {
  523. callbackRes, err := ioutil.ReadAll(res.Body)
  524. if err != nil {
  525. return nil, &TosServerError{
  526. TosError: TosError{Message: fmt.Sprintf("tos: read callback result err:%s", err.Error())},
  527. RequestInfo: res.RequestInfo(),
  528. }
  529. }
  530. if len(callbackRes) > 0 {
  531. callbackResult = string(callbackRes)
  532. }
  533. }
  534. return &PutObjectV2Output{
  535. RequestInfo: res.RequestInfo(),
  536. ETag: res.Header.Get(HeaderETag),
  537. SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
  538. SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
  539. VersionID: res.Header.Get(HeaderVersionID),
  540. ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption),
  541. ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID),
  542. CallbackResult: callbackResult,
  543. HashCrc64ecma: crc64,
  544. }, nil
  545. }
  546. // PutObjectFromFile put an object from file
  547. func (cli *ClientV2) PutObjectFromFile(ctx context.Context, input *PutObjectFromFileInput) (*PutObjectFromFileOutput, error) {
  548. file, err := os.Open(input.FilePath)
  549. if err != nil {
  550. return nil, err
  551. }
  552. defer file.Close()
  553. putOutput, err := cli.PutObjectV2(ctx, &PutObjectV2Input{
  554. PutObjectBasicInput: input.PutObjectBasicInput,
  555. Content: file,
  556. })
  557. if err != nil {
  558. return nil, err
  559. }
  560. return &PutObjectFromFileOutput{*putOutput}, err
  561. }
  562. // AppendObject append content at the tail of an appendable object
  563. // objectKey: the name of object
  564. // content: the content of object
  565. // offset: append position, equals to the current object-size
  566. // options: WithContentType set Content-Type,
  567. // WithContentDisposition set Content-Disposition,
  568. // WithContentLanguage set Content-Language,
  569. // WithContentEncoding set Content-Encoding,
  570. // WithCacheControl set Cache-Control,
  571. // WithExpires set Expires,
  572. // WithMeta set meta header(s),
  573. // WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl
  574. // above options only take effect when offset parameter is 0.
  575. // WithContentSHA256 set Content-Sha256,
  576. // WithContentMD5 set Content-MD5.
  577. //
  578. // NOTICE: only content with a known length is supported now,
  579. // e.g, bytes.Buffer, bytes.Reader, strings.Reader, os.File, io.LimitedReader, net.Buffers.
  580. // if the parameter content(an io.Reader) is not one of these,
  581. // please use io.LimitReader(reader, length) to wrap this reader or use the WithContentLength option.
  582. //
  583. // Deprecated: use AppendObject of ClientV2 instead
  584. func (bkt *Bucket) AppendObject(ctx context.Context, objectKey string, content io.Reader, offset int64, options ...Option) (*AppendObjectOutput, error) {
  585. if err := isValidKey(objectKey); err != nil {
  586. return nil, err
  587. }
  588. res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).
  589. WithQuery("append", "").
  590. WithQuery("offset", strconv.FormatInt(offset, 10)).
  591. WithRetry(nil, NoRetryClassifier{}).
  592. Request(ctx, http.MethodPost, content, bkt.client.roundTripper(http.StatusOK))
  593. if err != nil {
  594. return nil, err
  595. }
  596. defer res.Close()
  597. nextOffset := res.Header.Get(HeaderNextAppendOffset)
  598. appendOffset, err := strconv.ParseInt(nextOffset, 10, 64)
  599. if err != nil {
  600. return nil, fmt.Errorf("tos: server return unexpected Next-Append-Offset header %q", nextOffset)
  601. }
  602. return &AppendObjectOutput{
  603. RequestInfo: res.RequestInfo(),
  604. ETag: res.Header.Get(HeaderETag),
  605. NextAppendOffset: appendOffset,
  606. }, nil
  607. }
  608. func (bkt *Bucket) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput, option ...Option) (*PutObjectTaggingOutput, error) {
  609. return bkt.baseClient.PutObjectTagging(ctx, input, option...)
  610. }
  611. func (bkt *Bucket) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput, option ...Option) (*GetObjectTaggingOutput, error) {
  612. return bkt.baseClient.GetObjectTagging(ctx, input, option...)
  613. }
  614. func (bkt *Bucket) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput, option ...Option) (*DeleteObjectTaggingOutput, error) {
  615. return bkt.baseClient.DeleteObjectTagging(ctx, input, option...)
  616. }
  617. func (bkt *Bucket) RestoreObject(ctx context.Context, input *RestoreObjectInput, option ...Option) (*RestoreObjectOutput, error) {
  618. return bkt.baseClient.RestoreObject(ctx, input, option...)
  619. }
  620. // AppendObjectV2 append content at the tail of an appendable object
  621. func (cli *ClientV2) AppendObjectV2(ctx context.Context, input *AppendObjectV2Input) (*AppendObjectV2Output, error) {
  622. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  623. return nil, err
  624. }
  625. var (
  626. checker hash.Hash64
  627. content = input.Content
  628. contentLength = input.ContentLength
  629. )
  630. if contentLength <= 0 {
  631. contentLength = tryResolveLength(content)
  632. }
  633. if cli.enableCRC {
  634. checker = NewCRC(DefaultCrcTable(), input.PreHashCrc64ecma)
  635. }
  636. if content != nil {
  637. content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker})
  638. }
  639. res, err := cli.newBuilder(input.Bucket, input.Key).
  640. WithQuery("append", "").
  641. WithParams(*input).
  642. WithContentLength(contentLength).
  643. WithRetry(nil, NoRetryClassifier{}).
  644. Request(ctx, http.MethodPost, content, cli.roundTripper(http.StatusOK))
  645. if err != nil {
  646. return nil, err
  647. }
  648. defer res.Close()
  649. nextOffset := res.Header.Get(HeaderNextAppendOffset)
  650. appendOffset, err := strconv.ParseInt(nextOffset, 10, 64)
  651. if err != nil {
  652. return nil, &TosServerError{
  653. TosError: TosError{fmt.Sprintf("tos: server return unexpected Next-Append-Offset header %q", nextOffset)},
  654. RequestInfo: res.RequestInfo(),
  655. }
  656. }
  657. if err = checkCrc64(res, checker); err != nil {
  658. return nil, err
  659. }
  660. crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
  661. return &AppendObjectV2Output{
  662. RequestInfo: res.RequestInfo(),
  663. VersionID: res.Header.Get(HeaderVersionID),
  664. NextAppendOffset: appendOffset,
  665. HashCrc64ecma: crc64,
  666. }, nil
  667. }
  668. // SetObjectMeta overwrites metadata of the object
  669. // objectKey: the name of object
  670. // options: WithContentType set Content-Type,
  671. // WithContentDisposition set Content-Disposition,
  672. // WithContentLanguage set Content-Language,
  673. // WithContentEncoding set Content-Encoding,
  674. // WithCacheControl set Cache-Control,
  675. // WithExpires set Expires,
  676. // WithMeta set meta header(s),
  677. // WithVersionID which version of this object will be set
  678. //
  679. // NOTICE: SetObjectMeta always overwrites all previous metadata
  680. //
  681. // Deprecated: use SetObjectMeta of ClientV2 instead
  682. func (bkt *Bucket) SetObjectMeta(ctx context.Context, objectKey string, options ...Option) (*SetObjectMetaOutput, error) {
  683. if err := isValidKey(objectKey); err != nil {
  684. return nil, err
  685. }
  686. res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).
  687. WithQuery("metadata", "").
  688. WithRetry(nil, StatusCodeClassifier{}).
  689. Request(ctx, http.MethodPost, nil, bkt.client.roundTripper(http.StatusOK))
  690. if err != nil {
  691. return nil, err
  692. }
  693. defer res.Close()
  694. return &SetObjectMetaOutput{RequestInfo: res.RequestInfo()}, nil
  695. }
  696. // SetObjectMeta overwrites metadata of the object
  697. func (cli *ClientV2) SetObjectMeta(ctx context.Context, input *SetObjectMetaInput) (*SetObjectMetaOutput, error) {
  698. if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
  699. return nil, err
  700. }
  701. res, err := cli.newBuilder(input.Bucket, input.Key).
  702. WithQuery("metadata", "").
  703. WithParams(*input).
  704. WithRetry(nil, StatusCodeClassifier{}).
  705. Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK))
  706. if err != nil {
  707. return nil, err
  708. }
  709. defer res.Close()
  710. return &SetObjectMetaOutput{RequestInfo: res.RequestInfo()}, nil
  711. }
  712. // ListObjects list objects of a bucket
  713. //
  714. // Deprecated: use ListObjectsV2 of ClientV2 instead
  715. func (bkt *Bucket) ListObjects(ctx context.Context, input *ListObjectsInput, options ...Option) (*ListObjectsOutput, error) {
  716. res, err := bkt.client.newBuilder(bkt.name, "", options...).
  717. WithQuery("prefix", input.Prefix).
  718. WithQuery("delimiter", input.Delimiter).
  719. WithQuery("marker", input.Marker).
  720. WithQuery("max-keys", strconv.Itoa(input.MaxKeys)).
  721. WithQuery("encoding-type", input.EncodingType).
  722. WithQuery("fetch-meta", strconv.FormatBool(input.FetchMeta)).
  723. WithRetry(nil, StatusCodeClassifier{}).
  724. Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
  725. if err != nil {
  726. return nil, err
  727. }
  728. defer res.Close()
  729. internalOutput := &listObjectsOutput{}
  730. if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &internalOutput); err != nil {
  731. return nil, err
  732. }
  733. output := ListObjectsOutput{
  734. RequestInfo: res.RequestInfo(),
  735. Name: internalOutput.Name,
  736. Prefix: internalOutput.Prefix,
  737. Marker: internalOutput.Marker,
  738. MaxKeys: internalOutput.MaxKeys,
  739. NextMarker: internalOutput.NextMarker,
  740. Delimiter: internalOutput.Delimiter,
  741. IsTruncated: internalOutput.IsTruncated,
  742. EncodingType: internalOutput.EncodingType,
  743. CommonPrefixes: internalOutput.CommonPrefixes,
  744. Contents: nil,
  745. }
  746. contents := make([]ListedObject, 0, len(internalOutput.Contents))
  747. for _, content := range internalOutput.Contents {
  748. contents = append(contents, ListedObject{
  749. Key: content.Key,
  750. LastModified: content.LastModified,
  751. ETag: content.ETag,
  752. Size: content.Size,
  753. Owner: content.Owner,
  754. StorageClass: content.StorageClass,
  755. Type: content.Type,
  756. Meta: parseUserMetaData(content.Meta),
  757. })
  758. }
  759. output.Contents = contents
  760. return &output, nil
  761. }
  762. // ListObjectsV2 list objects of a bucket
  763. // Deprecated: use ListObjectsType2 of ClientV2 instead
  764. func (cli *ClientV2) ListObjectsV2(ctx context.Context, input *ListObjectsV2Input) (*ListObjectsV2Output, error) {
  765. if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
  766. return nil, err
  767. }
  768. res, err := cli.newBuilder(input.Bucket, "").
  769. WithParams(*input).
  770. WithRetry(nil, StatusCodeClassifier{}).
  771. Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
  772. if err != nil {
  773. return nil, err
  774. }
  775. defer res.Close()
  776. temp := listObjectsV2Output{
  777. RequestInfo: res.RequestInfo(),
  778. }
  779. if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
  780. return nil, err
  781. }
  782. contents := make([]ListedObjectV2, 0, len(temp.Contents))
  783. for _, object := range temp.Contents {
  784. var hashCrc uint64
  785. if len(object.HashCrc64ecma) == 0 {
  786. hashCrc = 0
  787. } else {
  788. hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64)
  789. if err != nil {
  790. return nil, &TosServerError{
  791. TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
  792. RequestInfo: RequestInfo{RequestID: temp.RequestID},
  793. }
  794. }
  795. }
  796. contents = append(contents, ListedObjectV2{
  797. Key: object.Key,
  798. LastModified: object.LastModified,
  799. ETag: object.ETag,
  800. Size: object.Size,
  801. Owner: object.Owner,
  802. StorageClass: object.StorageClass,
  803. HashCrc64ecma: uint64(hashCrc),
  804. Meta: parseUserMetaData(object.Meta),
  805. })
  806. }
  807. output := ListObjectsV2Output{
  808. RequestInfo: temp.RequestInfo,
  809. Name: temp.Name,
  810. Prefix: temp.Prefix,
  811. Marker: temp.Marker,
  812. MaxKeys: temp.MaxKeys,
  813. NextMarker: temp.NextMarker,
  814. Delimiter: temp.Delimiter,
  815. IsTruncated: temp.IsTruncated,
  816. EncodingType: temp.EncodingType,
  817. CommonPrefixes: temp.CommonPrefixes,
  818. Contents: contents,
  819. }
  820. return &output, nil
  821. }
  822. func (cli *ClientV2) listObjectsType2(ctx context.Context, input *ListObjectsType2Input) (*ListObjectsType2Output, error) {
  823. res, err := cli.newBuilder(input.Bucket, "").
  824. WithParams(*input).
  825. WithQuery("list-type", "2").
  826. WithQuery("fetch-owner", "true").
  827. WithRetry(nil, StatusCodeClassifier{}).
  828. Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
  829. if err != nil {
  830. return nil, err
  831. }
  832. defer res.Close()
  833. temp := listObjectsType2Output{
  834. RequestInfo: res.RequestInfo(),
  835. }
  836. if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
  837. return nil, err
  838. }
  839. contents := make([]ListedObjectV2, 0, len(temp.Contents))
  840. for _, object := range temp.Contents {
  841. var hashCrc uint64
  842. if len(object.HashCrc64ecma) == 0 {
  843. hashCrc = 0
  844. } else {
  845. hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64)
  846. if err != nil {
  847. return nil, &TosServerError{
  848. TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
  849. RequestInfo: RequestInfo{RequestID: temp.RequestID},
  850. }
  851. }
  852. }
  853. contents = append(contents, ListedObjectV2{
  854. Key: object.Key,
  855. LastModified: object.LastModified,
  856. ETag: object.ETag,
  857. Size: object.Size,
  858. Owner: object.Owner,
  859. StorageClass: object.StorageClass,
  860. HashCrc64ecma: hashCrc,
  861. Meta: parseUserMetaData(object.Meta),
  862. })
  863. }
  864. output := ListObjectsType2Output{
  865. RequestInfo: temp.RequestInfo,
  866. Name: temp.Name,
  867. ContinuationToken: temp.ContinuationToken,
  868. Prefix: temp.Prefix,
  869. MaxKeys: temp.MaxKeys,
  870. KeyCount: temp.KeyCount,
  871. Delimiter: temp.Delimiter,
  872. IsTruncated: temp.IsTruncated,
  873. EncodingType: temp.EncodingType,
  874. CommonPrefixes: temp.CommonPrefixes,
  875. NextContinuationToken: temp.NextContinuationToken,
  876. Contents: contents,
  877. }
  878. return &output, nil
  879. }
  880. func (cli *ClientV2) ListObjectsType2(ctx context.Context, input *ListObjectsType2Input) (*ListObjectsType2Output, error) {
  881. if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
  882. return nil, err
  883. }
  884. copyInput := *input
  885. input = &copyInput
  886. if input.MaxKeys == 0 {
  887. input.MaxKeys = DefaultListMaxKeys
  888. }
  889. if input.ListOnlyOnce {
  890. return cli.listObjectsType2(ctx, input)
  891. }
  892. var output *ListObjectsType2Output
  893. for {
  894. res, err := cli.listObjectsType2(ctx, input)
  895. if err != nil {
  896. return nil, err
  897. }
  898. if output == nil {
  899. output = res
  900. } else {
  901. output.KeyCount += res.KeyCount
  902. output.IsTruncated = res.IsTruncated
  903. output.NextContinuationToken = res.NextContinuationToken
  904. output.Contents = append(output.Contents, res.Contents...)
  905. output.CommonPrefixes = append(output.CommonPrefixes, res.CommonPrefixes...)
  906. }
  907. if !res.IsTruncated || len(res.Contents) >= input.MaxKeys {
  908. break
  909. }
  910. input.ContinuationToken = res.NextContinuationToken
  911. input.MaxKeys = input.MaxKeys - res.KeyCount
  912. }
  913. return output, nil
  914. }
  915. // ListObjectVersions list multi-version objects of a bucket
  916. //
  917. // Deprecated: use ListObjectV2Versions of ClientV2 instead
  918. func (bkt *Bucket) ListObjectVersions(ctx context.Context, input *ListObjectVersionsInput, options ...Option) (*ListObjectVersionsOutput, error) {
  919. res, err := bkt.client.newBuilder(bkt.name, "", options...).
  920. WithQuery("prefix", input.Prefix).
  921. WithQuery("delimiter", input.Delimiter).
  922. WithQuery("key-marker", input.KeyMarker).
  923. WithQuery("max-keys", strconv.Itoa(input.MaxKeys)).
  924. WithQuery("encoding-type", input.EncodingType).
  925. WithQuery("fetch-meta", strconv.FormatBool(input.FetchMeta)).
  926. WithQuery("versions", "").
  927. WithRetry(nil, StatusCodeClassifier{}).
  928. Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
  929. if err != nil {
  930. return nil, err
  931. }
  932. defer res.Close()
  933. interOutput := listObjectVersionsOutput{RequestInfo: res.RequestInfo()}
  934. if err = marshalOutput(interOutput.RequestID, res.Body, &interOutput); err != nil {
  935. return nil, err
  936. }
  937. output := ListObjectVersionsOutput{
  938. RequestInfo: interOutput.RequestInfo,
  939. Name: interOutput.Name,
  940. Prefix: interOutput.Prefix,
  941. KeyMarker: interOutput.KeyMarker,
  942. VersionIDMarker: interOutput.VersionIDMarker,
  943. Delimiter: interOutput.Delimiter,
  944. EncodingType: interOutput.EncodingType,
  945. MaxKeys: interOutput.MaxKeys,
  946. NextKeyMarker: interOutput.NextKeyMarker,
  947. NextVersionIDMarker: interOutput.NextVersionIDMarker,
  948. IsTruncated: interOutput.IsTruncated,
  949. CommonPrefixes: interOutput.CommonPrefixes,
  950. DeleteMarkers: interOutput.DeleteMarkers,
  951. }
  952. contents := make([]ListedObjectVersion, 0, len(interOutput.Versions))
  953. for _, content := range interOutput.Versions {
  954. contents = append(contents, ListedObjectVersion{
  955. Key: content.Key,
  956. IsLatest: content.IsLatest,
  957. LastModified: content.LastModified,
  958. ETag: content.ETag,
  959. Size: content.Size,
  960. Owner: content.Owner,
  961. StorageClass: content.StorageClass,
  962. Type: content.Type,
  963. VersionID: content.VersionID,
  964. Meta: parseUserMetaData(content.Meta),
  965. })
  966. }
  967. output.Versions = contents
  968. return &output, nil
  969. }
  970. // ListObjectVersionsV2 list multi-version objects of a bucket
  971. func (cli *ClientV2) ListObjectVersionsV2(
  972. ctx context.Context,
  973. input *ListObjectVersionsV2Input) (*ListObjectVersionsV2Output, error) {
  974. if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
  975. return nil, err
  976. }
  977. res, err := cli.newBuilder(input.Bucket, "").
  978. WithParams(*input).
  979. WithQuery("versions", "").
  980. WithRetry(nil, StatusCodeClassifier{}).
  981. Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
  982. if err != nil {
  983. return nil, err
  984. }
  985. defer res.Close()
  986. temp := listObjectVersionsV2Output{RequestInfo: res.RequestInfo()}
  987. if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
  988. return nil, err
  989. }
  990. versions := make([]ListedObjectVersionV2, 0, len(temp.Versions))
  991. for _, version := range temp.Versions {
  992. var hashCrc uint64
  993. if len(version.HashCrc64ecma) == 0 {
  994. hashCrc = 0
  995. } else {
  996. hashCrc, err = strconv.ParseUint(version.HashCrc64ecma, 10, 64)
  997. if err != nil {
  998. return nil, &TosServerError{
  999. TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
  1000. RequestInfo: RequestInfo{RequestID: temp.RequestID},
  1001. }
  1002. }
  1003. }
  1004. versions = append(versions, ListedObjectVersionV2{
  1005. Key: version.Key,
  1006. LastModified: version.LastModified,
  1007. ETag: version.ETag,
  1008. IsLatest: version.IsLatest,
  1009. Size: version.Size,
  1010. Owner: version.Owner,
  1011. StorageClass: version.StorageClass,
  1012. VersionID: version.VersionID,
  1013. HashCrc64ecma: hashCrc,
  1014. Meta: parseUserMetaData(version.Meta),
  1015. })
  1016. }
  1017. output := ListObjectVersionsV2Output{
  1018. RequestInfo: temp.RequestInfo,
  1019. Name: temp.Name,
  1020. Prefix: temp.Prefix,
  1021. KeyMarker: temp.KeyMarker,
  1022. VersionIDMarker: temp.VersionIDMarker,
  1023. Delimiter: temp.Delimiter,
  1024. EncodingType: temp.EncodingType,
  1025. MaxKeys: temp.MaxKeys,
  1026. NextKeyMarker: temp.NextKeyMarker,
  1027. NextVersionIDMarker: temp.NextVersionIDMarker,
  1028. IsTruncated: temp.IsTruncated,
  1029. CommonPrefixes: temp.CommonPrefixes,
  1030. DeleteMarkers: temp.DeleteMarkers,
  1031. Versions: versions,
  1032. }
  1033. return &output, nil
  1034. }
  1035. func (cli *ClientV2) RestoreObject(ctx context.Context, input *RestoreObjectInput) (*RestoreObjectOutput, error) {
  1036. return cli.baseClient.RestoreObject(ctx, input)
  1037. }