Skip to content

Commit 933d18a

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat: expose response headers for both streams and errors
1 parent 69c97ad commit 933d18a

File tree

9 files changed

+80
-29
lines changed

9 files changed

+80
-29
lines changed

‎lib/scrapegraphai/errors.rb‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,20 +40,25 @@ class APIError < Scrapegraphai::Errors::Error
4040
# @return [Integer, nil]
4141
attr_accessor :status
4242

43+
# @return [Hash{String=>String}, nil]
44+
attr_accessor :headers
45+
4346
# @return [Object, nil]
4447
attr_accessor :body
4548

4649
# @api private
4750
#
4851
# @param url [URI::Generic]
4952
# @param status [Integer, nil]
53+
# @param headers [Hash{String=>String}, nil]
5054
# @param body [Object, nil]
5155
# @param request [nil]
5256
# @param response [nil]
5357
# @param message [String, nil]
54-
def initialize(url:, status: nil, body: nil, request: nil, response: nil, message: nil)
58+
def initialize(url:, status: nil, headers: nil, body: nil, request: nil, response: nil, message: nil)
5559
@url = url
5660
@status = status
61+
@headers = headers
5762
@body = body
5863
@request = request
5964
@response = response
@@ -74,13 +79,15 @@ class APIConnectionError < Scrapegraphai::Errors::APIError
7479
#
7580
# @param url [URI::Generic]
7681
# @param status [nil]
82+
# @param headers [Hash{String=>String}, nil]
7783
# @param body [nil]
7884
# @param request [nil]
7985
# @param response [nil]
8086
# @param message [String, nil]
8187
def initialize(
8288
url:,
8389
status: nil,
90+
headers: nil,
8491
body: nil,
8592
request: nil,
8693
response: nil,
@@ -95,13 +102,15 @@ class APITimeoutError < Scrapegraphai::Errors::APIConnectionError
95102
#
96103
# @param url [URI::Generic]
97104
# @param status [nil]
105+
# @param headers [Hash{String=>String}, nil]
98106
# @param body [nil]
99107
# @param request [nil]
100108
# @param response [nil]
101109
# @param message [String, nil]
102110
def initialize(
103111
url:,
104112
status: nil,
113+
headers: nil,
105114
body: nil,
106115
request: nil,
107116
response: nil,
@@ -116,21 +125,24 @@ class APIStatusError < Scrapegraphai::Errors::APIError
116125
#
117126
# @param url [URI::Generic]
118127
# @param status [Integer]
128+
# @param headers [Hash{String=>String}, nil]
119129
# @param body [Object, nil]
120130
# @param request [nil]
121131
# @param response [nil]
122132
# @param message [String, nil]
123133
#
124134
# @return [self]
125-
def self.for(url:, status:, body:, request:, response:, message: nil)
126-
kwargs = {
127-
url: url,
128-
status: status,
129-
body: body,
130-
request: request,
131-
response: response,
132-
message: message
133-
}
135+
def self.for(url:, status:, headers:, body:, request:, response:, message: nil)
136+
kwargs =
137+
{
138+
url: url,
139+
status: status,
140+
headers: headers,
141+
body: body,
142+
request: request,
143+
response: response,
144+
message: message
145+
}
134146

135147
case status
136148
in 400
@@ -162,15 +174,17 @@ def self.for(url:, status:, body:, request:, response:, message: nil)
162174
#
163175
# @param url [URI::Generic]
164176
# @param status [Integer]
177+
# @param headers [Hash{String=>String}, nil]
165178
# @param body [Object, nil]
166179
# @param request [nil]
167180
# @param response [nil]
168181
# @param message [String, nil]
169-
def initialize(url:, status:, body:, request:, response:, message: nil)
182+
def initialize(url:, status:, headers:, body:, request:, response:, message: nil)
170183
message ||= {url: url.to_s, status: status, body: body}
171184
super(
172185
url: url,
173186
status: status,
187+
headers: headers,
174188
body: body,
175189
request: request,
176190
response: response,

‎lib/scrapegraphai/internal/transport/base_client.rb‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ def validate!(req)
4747
# @api private
4848
#
4949
# @param status [Integer]
50-
# @param headers [Hash{String=>String}, Net::HTTPHeader]
50+
# @param headers [Hash{String=>String}]
5151
#
5252
# @return [Boolean]
5353
def should_retry?(status, headers:)
@@ -85,7 +85,7 @@ def should_retry?(status, headers:)
8585
#
8686
# @param status [Integer]
8787
#
88-
# @param response_headers [Hash{String=>String}, Net::HTTPHeader]
88+
# @param response_headers [Hash{String=>String}]
8989
#
9090
# @return [Hash{Symbol=>Object}]
9191
def follow_redirect(request, status:, response_headers:)
@@ -378,6 +378,7 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:)
378378
rescue Scrapegraphai::Errors::APIConnectionError => e
379379
status = e
380380
end
381+
headers = Scrapegraphai::Internal::Util.normalized_headers(response&.each_header&.to_h)
381382

382383
case status
383384
in ..299
@@ -394,7 +395,7 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:)
394395
in 300..399
395396
self.class.reap_connection!(status, stream: stream)
396397

397-
request = self.class.follow_redirect(request, status: status, response_headers: response)
398+
request = self.class.follow_redirect(request, status: status, response_headers: headers)
398399
send_request(
399400
request,
400401
redirect_count: redirect_count + 1,
@@ -403,16 +404,17 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:)
403404
)
404405
in Scrapegraphai::Errors::APIConnectionError if retry_count >= max_retries
405406
raise status
406-
in (400..) if retry_count >= max_retries || !self.class.should_retry?(status, headers: response)
407+
in (400..) if retry_count >= max_retries || !self.class.should_retry?(status, headers: headers)
407408
decoded = Kernel.then do
408-
Scrapegraphai::Internal::Util.decode_content(response, stream: stream, suppress_error: true)
409+
Scrapegraphai::Internal::Util.decode_content(headers, stream: stream, suppress_error: true)
409410
ensure
410411
self.class.reap_connection!(status, stream: stream)
411412
end
412413

413414
raise Scrapegraphai::Errors::APIStatusError.for(
414415
url: url,
415416
status: status,
417+
headers: headers,
416418
body: decoded,
417419
request: nil,
418420
response: response
@@ -489,19 +491,21 @@ def request(req)
489491
send_retry_header: send_retry_header
490492
)
491493

492-
decoded = Scrapegraphai::Internal::Util.decode_content(response, stream: stream)
494+
headers = Scrapegraphai::Internal::Util.normalized_headers(response.each_header.to_h)
495+
decoded = Scrapegraphai::Internal::Util.decode_content(headers, stream: stream)
493496
case req
494497
in {stream: Class => st}
495498
st.new(
496499
model: model,
497500
url: url,
498501
status: status,
502+
headers: headers,
499503
response: response,
500504
unwrap: unwrap,
501505
stream: decoded
502506
)
503507
in {page: Class => page}
504-
page.new(client: self, req: req, headers: response, page_data: decoded)
508+
page.new(client: self, req: req, headers: headers, page_data: decoded)
505509
else
506510
unwrapped = Scrapegraphai::Internal::Util.dig(decoded, unwrap)
507511
Scrapegraphai::Internal::Type::Converter.coerce(model, unwrapped)

‎lib/scrapegraphai/internal/type/base_page.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def to_enum = super(:auto_paging_each)
3939
#
4040
# @param client [Scrapegraphai::Internal::Transport::BaseClient]
4141
# @param req [Hash{Symbol=>Object}]
42-
# @param headers [Hash{String=>String}, Net::HTTPHeader]
42+
# @param headers [Hash{String=>String}]
4343
# @param page_data [Object]
4444
def initialize(client:, req:, headers:, page_data:)
4545
@client = client

‎lib/scrapegraphai/internal/util.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -647,7 +647,7 @@ def force_charset!(content_type, text:)
647647
#
648648
# Assumes each chunk in stream has `Encoding::BINARY`.
649649
#
650-
# @param headers [Hash{String=>String}, Net::HTTPHeader]
650+
# @param headers [Hash{String=>String}]
651651
# @param stream [Enumerable<String>]
652652
# @param suppress_error [Boolean]
653653
#

‎rbi/scrapegraphai/errors.rbi‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ module Scrapegraphai
3333
sig { returns(T.nilable(Integer)) }
3434
attr_accessor :status
3535

36+
sig { returns(T.nilable(T::Hash[String, String])) }
37+
attr_accessor :headers
38+
3639
sig { returns(T.nilable(T.anything)) }
3740
attr_accessor :body
3841

@@ -41,6 +44,7 @@ module Scrapegraphai
4144
params(
4245
url: URI::Generic,
4346
status: T.nilable(Integer),
47+
headers: T.nilable(T::Hash[String, String]),
4448
body: T.nilable(Object),
4549
request: NilClass,
4650
response: NilClass,
@@ -50,6 +54,7 @@ module Scrapegraphai
5054
def self.new(
5155
url:,
5256
status: nil,
57+
headers: nil,
5358
body: nil,
5459
request: nil,
5560
response: nil,
@@ -70,6 +75,7 @@ module Scrapegraphai
7075
params(
7176
url: URI::Generic,
7277
status: NilClass,
78+
headers: T.nilable(T::Hash[String, String]),
7379
body: NilClass,
7480
request: NilClass,
7581
response: NilClass,
@@ -79,6 +85,7 @@ module Scrapegraphai
7985
def self.new(
8086
url:,
8187
status: nil,
88+
headers: nil,
8289
body: nil,
8390
request: nil,
8491
response: nil,
@@ -93,6 +100,7 @@ module Scrapegraphai
93100
params(
94101
url: URI::Generic,
95102
status: NilClass,
103+
headers: T.nilable(T::Hash[String, String]),
96104
body: NilClass,
97105
request: NilClass,
98106
response: NilClass,
@@ -102,6 +110,7 @@ module Scrapegraphai
102110
def self.new(
103111
url:,
104112
status: nil,
113+
headers: nil,
105114
body: nil,
106115
request: nil,
107116
response: nil,
@@ -116,13 +125,22 @@ module Scrapegraphai
116125
params(
117126
url: URI::Generic,
118127
status: Integer,
128+
headers: T.nilable(T::Hash[String, String]),
119129
body: T.nilable(Object),
120130
request: NilClass,
121131
response: NilClass,
122132
message: T.nilable(String)
123133
).returns(T.attached_class)
124134
end
125-
def self.for(url:, status:, body:, request:, response:, message: nil)
135+
def self.for(
136+
url:,
137+
status:,
138+
headers:,
139+
body:,
140+
request:,
141+
response:,
142+
message: nil
143+
)
126144
end
127145

128146
sig { returns(Integer) }
@@ -133,13 +151,22 @@ module Scrapegraphai
133151
params(
134152
url: URI::Generic,
135153
status: Integer,
154+
headers: T.nilable(T::Hash[String, String]),
136155
body: T.nilable(Object),
137156
request: NilClass,
138157
response: NilClass,
139158
message: T.nilable(String)
140159
).returns(T.attached_class)
141160
end
142-
def self.new(url:, status:, body:, request:, response:, message: nil)
161+
def self.new(
162+
url:,
163+
status:,
164+
headers:,
165+
body:,
166+
request:,
167+
response:,
168+
message: nil
169+
)
143170
end
144171
end
145172

‎rbi/scrapegraphai/internal/transport/base_client.rbi‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,9 @@ module Scrapegraphai
8585

8686
# @api private
8787
sig do
88-
params(
89-
status: Integer,
90-
headers: T.any(T::Hash[String, String], Net::HTTPHeader)
91-
).returns(T::Boolean)
88+
params(status: Integer, headers: T::Hash[String, String]).returns(
89+
T::Boolean
90+
)
9291
end
9392
def should_retry?(status, headers:)
9493
end
@@ -99,7 +98,7 @@ module Scrapegraphai
9998
request:
10099
Scrapegraphai::Internal::Transport::BaseClient::RequestInput,
101100
status: Integer,
102-
response_headers: T.any(T::Hash[String, String], Net::HTTPHeader)
101+
response_headers: T::Hash[String, String]
103102
).returns(
104103
Scrapegraphai::Internal::Transport::BaseClient::RequestInput
105104
)

‎rbi/scrapegraphai/internal/type/base_page.rbi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ module Scrapegraphai
3131
client: Scrapegraphai::Internal::Transport::BaseClient,
3232
req:
3333
Scrapegraphai::Internal::Transport::BaseClient::RequestComponents,
34-
headers: T.any(T::Hash[String, String], Net::HTTPHeader),
34+
headers: T::Hash[String, String],
3535
page_data: T.anything
3636
).void
3737
end

‎rbi/scrapegraphai/internal/util.rbi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ module Scrapegraphai
361361
# Assumes each chunk in stream has `Encoding::BINARY`.
362362
sig do
363363
params(
364-
headers: T.any(T::Hash[String, String], Net::HTTPHeader),
364+
headers: T::Hash[String, String],
365365
stream: T::Enumerable[String],
366366
suppress_error: T::Boolean
367367
).returns(T.anything)

0 commit comments

Comments
 (0)