GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

Consolidate workspace deps and migrate sha1 to sha2
Some checks failed
CI / verify (push) Has been cancelled
Details

Browse Source
This commit is contained in:
kdletters
2026-05-07 21:11:14 +08:00
parent 9146e5b8ec
commit df80876f60
41 changed files with 949 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
server-rs/crates/platform-oss/Cargo.toml
View File

@@ -5,14 +5,13 @@ version.workspace = true
license.workspace = true
[dependencies]
base64 = "0.22"
hmac = "0.12"
httpdate = "1"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha1 = "0.10"
time = { version = "0.3", features = ["formatting"] }
base64 = { workspace = true }
hmac = { workspace = true }
reqwest = { workspace = true, features = ["rustls-tls"] }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
time = { workspace = true, features = ["formatting"] }
[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt"] }
tokio = { workspace = true, features = ["macros", "rt"] }

348
server-rs/crates/platform-oss/src/lib.rs
View File

@@ -1,21 +1,24 @@
use std::{collections::BTreeMap, error::Error, fmt, time::SystemTime};
use std::{collections::BTreeMap, error::Error, fmt};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use hmac::{Hmac, Mac};
use httpdate::fmt_http_date;
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sha1::Sha1;
use sha2::{Digest, Sha256};
use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
type HmacSha1 = Hmac<Sha1>;
type HmacSha256 = Hmac<Sha256>;
pub const DEFAULT_POST_EXPIRE_SECONDS: u64 = 10 * 60;
pub const DEFAULT_READ_EXPIRE_SECONDS: u64 = 10 * 60;
pub const DEFAULT_POST_MAX_SIZE_BYTES: u64 = 20 * 1024 * 1024;
pub const DEFAULT_SUCCESS_ACTION_STATUS: u16 = 200;
pub const DEFAULT_METADATA_TOTAL_BYTES_LIMIT: usize = 8 * 1024;
const OSS_V4_ALGORITHM: &str = "OSS4-HMAC-SHA256";
const OSS_V4_REQUEST: &str = "aliyun_v4_request";
const OSS_V4_SERVICE: &str = "oss";
const OSS_UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
pub const LEGACY_PUBLIC_PREFIXES: [&str; 8] = [
"generated-character-drafts",
@@ -171,9 +174,13 @@ pub struct OssPutObjectResponse {
pub struct OssPostObjectFormFields {
pub key: String,
pub policy: String,
#[serde(rename = "OSSAccessKeyId")]
pub oss_access_key_id: String,
#[serde(rename = "Signature")]
#[serde(rename = "x-oss-signature-version")]
pub signature_version: String,
#[serde(rename = "x-oss-credential")]
pub credential: String,
#[serde(rename = "x-oss-date")]
pub date: String,
#[serde(rename = "x-oss-signature")]
pub signature: String,
#[serde(rename = "success_action_status")]
pub success_action_status: String,
@@ -394,6 +401,10 @@ impl OssClient {
.format(&Rfc3339)
.map_err(|error| OssError::SerializePolicy(format!("格式化过期时间失败：{error}")))?;
let signed_at = OffsetDateTime::now_utc();
let signature_scope = build_v4_signature_scope(&self.config.endpoint, signed_at)?;
let signature_date = build_v4_signature_date(signed_at)?;
let credential = format!("{}/{}", self.config.access_key_id, signature_scope);
let policy_json = build_policy_json(
&self.config.bucket,
&object_key,
@@ -402,14 +413,17 @@ impl OssClient {
success_action_status,
content_type.as_deref(),
&metadata,
&credential,
&signature_date,
);
let policy = serde_json::to_string(&policy_json)
.map_err(|error| OssError::SerializePolicy(format!("序列化 policy 失败：{error}")))?;
let encoded_policy = BASE64_STANDARD.encode(policy.as_bytes());
let signature = sign_policy(&self.config.access_key_secret, &encoded_policy)?;
let signature =
sign_v4_content(&self.config.access_key_secret, &signature_scope, &encoded_policy)?;
Ok(OssPostObjectResponse {
signature_version: "v1",
signature_version: "v4",
provider: "aliyun-oss",
bucket: self.config.bucket.clone(),
endpoint: self.config.endpoint.clone(),
@@ -425,7 +439,9 @@ impl OssClient {
form_fields: OssPostObjectFormFields {
key: object_key,
policy: encoded_policy,
oss_access_key_id: self.config.access_key_id.clone(),
signature_version: OSS_V4_ALGORITHM.to_string(),
credential,
date: signature_date,
signature,
success_action_status: success_action_status.to_string(),
content_type,
@@ -458,18 +474,48 @@ impl OssClient {
let expires_at_text = expires_at
.format(&Rfc3339)
.map_err(|error| OssError::Sign(format!("格式化过期时间失败：{error}")))?;
let expires_epoch_seconds = expires_at.unix_timestamp();
let canonical_resource = build_canonical_object_resource(&self.config.bucket, &object_key);
let string_to_sign = format!("GET\n\n\n{expires_epoch_seconds}\n{canonical_resource}");
let signature = sign_policy(&self.config.access_key_secret, &string_to_sign)?;
let signed_at = OffsetDateTime::now_utc();
let signed_at_text = build_v4_signature_date(signed_at)?;
let signature_scope = build_v4_signature_scope(&self.config.endpoint, signed_at)?;
let credential = format!("{}/{}", self.config.access_key_id, signature_scope);
let mut query = BTreeMap::from([
("x-oss-additional-headers".to_string(), "host".to_string()),
(
"x-oss-signature-version".to_string(),
OSS_V4_ALGORITHM.to_string(),
),
("x-oss-credential".to_string(), credential),
("x-oss-date".to_string(), signed_at_text),
("x-oss-expires".to_string(), expire_seconds.to_string()),
]);
let canonical_uri = build_v4_canonical_uri(&self.config.bucket, Some(&object_key));
let object_url_path = format!("/{}", encode_url_path(&object_key));
let additional_headers = "host";
let canonical_headers = format!(
"host:{}.{}\n",
self.config.bucket(),
self.config.endpoint()
);
let canonical_query = build_canonical_query_string(&query);
let canonical_request = build_v4_canonical_request(
Method::GET.as_str(),
&canonical_uri,
&canonical_query,
&canonical_headers,
additional_headers,
OSS_UNSIGNED_PAYLOAD,
);
let string_to_sign =
build_v4_string_to_sign(query["x-oss-date"].as_str(), &signature_scope, &canonical_request);
let signature =
sign_v4_content(&self.config.access_key_secret, &signature_scope, &string_to_sign)?;
query.insert("x-oss-signature".to_string(), signature);
let signed_url = format!(
"{}/{}?OSSAccessKeyId={}&Expires={}&Signature={}",
"{}{}?{}",
self.config.upload_host(),
encode_url_path(&object_key),
encode_url_query_value(&self.config.access_key_id),
expires_epoch_seconds,
encode_url_query_value(&signature)
object_url_path,
build_canonical_query_string(&query)
);
Ok(OssSignedGetObjectUrlResponse {
@@ -656,6 +702,8 @@ fn build_policy_json(
success_action_status: u16,
content_type: Option<&str>,
metadata: &BTreeMap<String, String>,
credential: &str,
signature_date: &str,
) -> Value {
let mut conditions = vec![
json!({ "bucket": bucket }),
@@ -666,6 +714,9 @@ fn build_policy_json(
"$success_action_status",
success_action_status.to_string()
]),
json!(["eq", "$x-oss-signature-version", OSS_V4_ALGORITHM]),
json!(["eq", "$x-oss-credential", credential]),
json!(["eq", "$x-oss-date", signature_date]),
];
if let Some(content_type) = content_type {
@@ -695,10 +746,6 @@ fn build_object_url(
Ok(url)
}
fn build_canonical_object_resource(bucket: &str, object_key: &str) -> String {
format!("/{bucket}/{object_key}")
}
fn build_object_key(
prefix: LegacyAssetPrefix,
path_segments: &[String],
@@ -928,14 +975,6 @@ fn collapse_dashes(value: &str) -> String {
.to_string()
}
fn sign_policy(access_key_secret: &str, encoded_policy: &str) -> Result<String, OssError> {
let mut signer = HmacSha1::new_from_slice(access_key_secret.as_bytes())
.map_err(|error| OssError::Sign(format!("初始化 HMAC-SHA1 失败：{error}")))?;
signer.update(encoded_policy.as_bytes());
Ok(BASE64_STANDARD.encode(signer.finalize().into_bytes()))
}
async fn send_signed_request(
client: &reqwest::Client,
config: &OssConfig,
@@ -966,29 +1005,52 @@ fn signed_request_builder(
content_type: Option<&str>,
oss_headers: &BTreeMap<String, String>,
) -> Result<reqwest::RequestBuilder, OssError> {
let date = fmt_http_date(SystemTime::now());
let canonical_resource = match object_key.map(str::trim).filter(|value| !value.is_empty()) {
Some(object_key) => {
build_canonical_object_resource(config.bucket(), object_key.trim_start_matches('/'))
}
None => format!("/{}/", config.bucket()),
};
let canonicalized_oss_headers = build_canonicalized_oss_headers(oss_headers);
let string_to_sign = format!(
"{}\n\n{}\n{}\n{}{}",
let signed_at = OffsetDateTime::now_utc();
let signed_at_text = build_v4_signature_date(signed_at)?;
let signature_scope = build_v4_signature_scope(config.endpoint(), signed_at)?;
let object_path = object_key.map(str::trim).filter(|value| !value.is_empty());
let canonical_uri = build_v4_canonical_uri(config.bucket(), object_path);
let body_sha256 = OSS_UNSIGNED_PAYLOAD.to_string();
let mut signed_headers = BTreeMap::from([
(
"host".to_string(),
format!("{}.{}", config.bucket(), config.endpoint()),
),
("x-oss-content-sha256".to_string(), body_sha256.clone()),
("x-oss-date".to_string(), signed_at_text.clone()),
]);
if let Some(content_type) = content_type {
signed_headers.insert("content-type".to_string(), content_type.to_string());
}
for (key, value) in oss_headers {
signed_headers.insert(key.to_ascii_lowercase(), value.trim().to_string());
}
let canonical_headers = build_v4_canonical_headers(&signed_headers);
let additional_headers = "host";
let canonical_request = build_v4_canonical_request(
method.as_str(),
content_type.unwrap_or_default(),
date,
canonicalized_oss_headers,
canonical_resource
&canonical_uri,
"",
&canonical_headers,
additional_headers,
&body_sha256,
);
let signature = sign_policy(config.access_key_secret(), &string_to_sign)?;
let string_to_sign = build_v4_string_to_sign(&signed_at_text, &signature_scope, &canonical_request);
let signature = sign_v4_content(config.access_key_secret(), &signature_scope, &string_to_sign)?;
let mut builder = client
.request(method, target_url)
.header("Date", date)
.header("x-oss-content-sha256", body_sha256)
.header("x-oss-date", signed_at_text)
.header(
"Authorization",
format!("OSS {}:{}", config.access_key_id(), signature),
format!(
"{OSS_V4_ALGORITHM} Credential={}/{},AdditionalHeaders={},Signature={}",
config.access_key_id(),
signature_scope,
additional_headers,
signature
),
);
if let Some(content_type) = content_type {
@@ -1002,13 +1064,160 @@ fn signed_request_builder(
Ok(builder)
}
fn build_canonicalized_oss_headers(headers: &BTreeMap<String, String>) -> String {
fn build_v4_signature_scope(endpoint: &str, signed_at: OffsetDateTime) -> Result<String, OssError> {
let date = signed_at
.date()
.to_string()
.replace('-', "");
let region = extract_oss_region(endpoint)?;
Ok(format!("{date}/{region}/{OSS_V4_SERVICE}/{OSS_V4_REQUEST}"))
}
fn build_v4_signature_date(signed_at: OffsetDateTime) -> Result<String, OssError> {
let date = signed_at
.date()
.to_string()
.replace('-', "");
let time = signed_at
.time()
.to_string()
.split('.')
.next()
.unwrap_or("00:00:00")
.replace(':', "");
if time.len() != 6 {
return Err(OssError::Sign("OSS V4 签名时间格式化失败".to_string()));
}
Ok(format!("{date}T{time}Z"))
}
fn build_v4_canonical_uri(bucket: &str, object_key: Option<&str>) -> String {
match object_key.map(str::trim).filter(|value| !value.is_empty()) {
Some(object_key) => format!(
"/{}/{}",
encode_url_query_value(bucket),
encode_url_path(object_key.trim_start_matches('/'))
),
None => format!("/{}/", encode_url_query_value(bucket)),
}
}
fn extract_oss_region(endpoint: &str) -> Result<String, OssError> {
endpoint
.trim()
.trim_start_matches("https://")
.trim_start_matches("http://")
.split('.')
.next()
.and_then(|segment| segment.strip_prefix("oss-"))
.map(str::to_string)
.filter(|region| !region.is_empty())
.ok_or_else(|| {
OssError::InvalidConfig(format!(
"OSS endpoint 无法解析 region，当前值：{endpoint}"
))
})
}
fn sign_v4_content(
access_key_secret: &str,
signature_scope: &str,
content: &str,
) -> Result<String, OssError> {
let signing_key = build_v4_signing_key(access_key_secret, signature_scope)?;
Ok(hex_sha256_hmac(&signing_key, content.as_bytes()))
}
fn build_v4_signing_key(access_key_secret: &str, signature_scope: &str) -> Result<Vec<u8>, OssError> {
let mut parts = signature_scope.split('/');
let date = parts
.next()
.ok_or_else(|| OssError::Sign("OSS V4 签名 scope 缺少日期".to_string()))?;
let region = parts
.next()
.ok_or_else(|| OssError::Sign("OSS V4 签名 scope 缺少 region".to_string()))?;
let service = parts
.next()
.ok_or_else(|| OssError::Sign("OSS V4 签名 scope 缺少 service".to_string()))?;
let request = parts
.next()
.ok_or_else(|| OssError::Sign("OSS V4 签名 scope 缺少 request".to_string()))?;
let date_key = hmac_sha256_raw(format!("aliyun_v4{access_key_secret}").as_bytes(), date)?;
let region_key = hmac_sha256_raw(&date_key, region)?;
let service_key = hmac_sha256_raw(&region_key, service)?;
hmac_sha256_raw(&service_key, request)
}
fn hmac_sha256_raw(key: &[u8], content: &str) -> Result<Vec<u8>, OssError> {
let mut signer = HmacSha256::new_from_slice(key)
.map_err(|error| OssError::Sign(format!("初始化 HMAC-SHA256 失败：{error}")))?;
signer.update(content.as_bytes());
Ok(signer.finalize().into_bytes().to_vec())
}
fn hex_sha256_hmac(key: &[u8], content: &[u8]) -> String {
let mut signer = HmacSha256::new_from_slice(key)
.expect("HMAC-SHA256 accepts keys of any size");
signer.update(content);
hex_lower(&signer.finalize().into_bytes())
}
fn build_v4_canonical_request(
method: &str,
canonical_uri: &str,
canonical_query: &str,
canonical_headers: &str,
signed_headers: &str,
payload_hash: &str,
) -> String {
format!(
"{method}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{payload_hash}"
)
}
fn build_v4_string_to_sign(
signature_date: &str,
signature_scope: &str,
canonical_request: &str,
) -> String {
format!(
"{OSS_V4_ALGORITHM}\n{signature_date}\n{signature_scope}\n{}",
sha256_hex(canonical_request.as_bytes())
)
}
fn sha256_hex(content: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(content);
hex_lower(&hasher.finalize())
}
fn hex_lower(bytes: &[u8]) -> String {
bytes
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
}
fn build_v4_canonical_headers(headers: &BTreeMap<String, String>) -> String {
headers
.iter()
.map(|(key, value)| format!("{}:{}\n", key.to_ascii_lowercase(), value.trim()))
.collect::<String>()
}
fn build_canonical_query_string(params: &BTreeMap<String, String>) -> String {
params
.iter()
.map(|(key, value)| format!("{}={}", encode_url_query_value(key), encode_url_query_value(value)))
.collect::<Vec<_>>()
.join("&")
}
fn encode_url_path(path: &str) -> String {
path.split('/')
.map(encode_url_query_value)
@@ -1115,8 +1324,20 @@ mod tests {
);
assert_eq!(response.bucket, "genarrative-assets".to_string());
assert_eq!(
response.form_fields.oss_access_key_id,
"test-access-key-id".to_string()
response.form_fields.signature_version,
OSS_V4_ALGORITHM.to_string()
);
assert!(response
.form_fields
.credential
.starts_with("test-access-key-id/"));
assert!(response
.form_fields
.credential
.ends_with("/cn-shanghai/oss/aliyun_v4_request"));
assert_eq!(
response.form_fields.date.len(),
"20260507T120000Z".len()
);
assert_eq!(
response.form_fields.metadata.get("x-oss-meta-asset-kind"),
@@ -1169,6 +1390,18 @@ mod tests {
);
assert_eq!(
policy["conditions"][4],
json!(["eq", "$x-oss-signature-version", "OSS4-HMAC-SHA256"])
);
assert_eq!(
policy["conditions"][5],
json!(["eq", "$x-oss-credential", response.form_fields.credential])
);
assert_eq!(
policy["conditions"][6],
json!(["eq", "$x-oss-date", response.form_fields.date])
);
assert_eq!(
policy["conditions"][7],
json!(["eq", "$content-type", "image/png"])
);
assert_eq!(response.bucket, "genarrative-assets".to_string());
@@ -1206,10 +1439,13 @@ mod tests {
assert!(
response
.signed_url
.contains("OSSAccessKeyId=test-access-key-id")
.contains("x-oss-signature-version=OSS4-HMAC-SHA256")
);
assert!(response.signed_url.contains("&Expires="));
assert!(response.signed_url.contains("&Signature="));
assert!(response
.signed_url
.contains("x-oss-credential=test-access-key-id%2F"));
assert!(response.signed_url.contains("&x-oss-expires=300"));
assert!(response.signed_url.contains("&x-oss-signature="));
}
#[test]
@@ -1282,7 +1518,7 @@ mod tests {
}
#[test]
fn canonicalized_oss_headers_matches_oss_v1_upload_signature_shape() {
fn canonicalized_oss_headers_matches_sorted_v4_header_shape() {
let headers = BTreeMap::from([
(
"x-oss-meta-source-job-id".to_string(),
@@ -1295,7 +1531,7 @@ mod tests {
]);
assert_eq!(
build_canonicalized_oss_headers(&headers),
build_v4_canonical_headers(&headers),
"x-oss-meta-asset-kind:character_visual\nx-oss-meta-source-job-id:job_001\n"
);
}