Gecko [ www.jdcpatala.org ]

Name	Size	Permission
antivirus.lua	11.24 KB	-rw-r--r--
arc.lua	26.72 KB	-rw-r--r--
asn.lua	4.78 KB	-rw-r--r--
aws_s3.lua	7.93 KB	-rw-r--r--
bayes_expiry.lua	14.21 KB	-rw-r--r--
bimi.lua	11.5 KB	-rw-r--r--
clickhouse.lua	49.6 KB	-rw-r--r--
clustering.lua	9.16 KB	-rw-r--r--
dcc.lua	3.19 KB	-rw-r--r--
dkim_signing.lua	5.25 KB	-rw-r--r--
dmarc.lua	21.89 KB	-rw-r--r--
dynamic_conf.lua	8.7 KB	-rw-r--r--
elastic.lua	54.28 KB	-rw-r--r--
emails.lua	128 B	-rw-r--r--
external_relay.lua	7.89 KB	-rw-r--r--
external_services.lua	12.3 KB	-rw-r--r--
force_actions.lua	7.67 KB	-rw-r--r--
forged_recipients.lua	5.32 KB	-rw-r--r--
fuzzy_collect.lua	5.46 KB	-rw-r--r--
gpt.lua	11.32 KB	-rw-r--r--
greylist.lua	16.36 KB	-rw-r--r--
hfilter.lua	16.15 KB	-rw-r--r--
history_redis.lua	9.31 KB	-rw-r--r--
http_headers.lua	6.46 KB	-rw-r--r--
ip_score.lua	130 B	-rw-r--r--
known_senders.lua	13.2 KB	-rw-r--r--
maillist.lua	6.55 KB	-rw-r--r--
maps_stats.lua	3.7 KB	-rw-r--r--
metadata_exporter.lua	20.19 KB	-rw-r--r--
metric_exporter.lua	6.4 KB	-rw-r--r--
mid.lua	3.58 KB	-rw-r--r--
milter_headers.lua	21 KB	-rw-r--r--
mime_types.lua	20.09 KB	-rw-r--r--
multimap.lua	40.08 KB	-rw-r--r--
mx_check.lua	11.42 KB	-rw-r--r--
neural.lua	32.82 KB	-rw-r--r--
once_received.lua	4.76 KB	-rw-r--r--
p0f.lua	2.88 KB	-rw-r--r--
phishing.lua	19.33 KB	-rw-r--r--
ratelimit.lua	22.77 KB	-rw-r--r--
rbl.lua	40.09 KB	-rw-r--r--
replies.lua	13.69 KB	-rw-r--r--
reputation.lua	39.69 KB	-rw-r--r--
rspamd_update.lua	4.07 KB	-rw-r--r--
settings.lua	40.17 KB	-rw-r--r--
spamassassin.lua	48.89 KB	-rw-r--r--
spamtrap.lua	5.87 KB	-rw-r--r--
spf.lua	7.49 KB	-rw-r--r--
trie.lua	4.89 KB	-rw-r--r--
url_redirector.lua	13.42 KB	-rw-r--r--
whitelist.lua	11.54 KB	-rw-r--r--

Code Editor : aws_s3.lua

--[[
Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--

local N = "aws_s3"
local lua_util = require "lua_util"
local lua_aws = require "lua_aws"
local rspamd_logger = require "rspamd_logger"
local ts = (require "tableshape").types
local rspamd_text = require "rspamd_text"
local rspamd_http = require "rspamd_http"
local rspamd_util = require "rspamd_util"

local settings = {
  s3_bucket = nil,
  s3_region = 'us-east-1',
  s3_host = 's3.amazonaws.com',
  s3_secret_key = nil,
  s3_key_id = nil,
  s3_timeout = 10,
  save_raw = true,
  save_structure = false,
  inline_content_limit = nil,
}

local settings_schema = ts.shape {
  s3_bucket = ts.string,
  s3_region = ts.string,
  s3_host = ts.string,
  s3_secret_key = ts.string,
  s3_key_id = ts.string,
  s3_timeout = ts.number + ts.string / lua_util.parse_time_interval,
  enabled = ts.boolean:is_optional(),
  fail_action = ts.string:is_optional(),
  zstd_compress = ts.boolean:is_optional(),
  save_raw = ts.boolean:is_optional(),
  save_structure = ts.boolean:is_optional(),
  inline_content_limit = ts.number:is_optional(),
}

local function raw_data(task, nonce, queue_id)
  local ext, content, content_type

if settings.zstd_compress then
    ext = 'eml.zst'
    content = rspamd_util.zstd_compress(task:get_content())
    content_type = 'application/zstd'
  else
    ext = 'eml'
    content = task:get_content()
    content_type = 'message/rfc-822'
  end

local path = string.format('/%s-%s.%s', queue_id, nonce, ext)

return path, content, content_type
end

local function gen_ext(base)
  local ext = base
  if settings.zstd_compress then
    ext = base .. '.zst'
  end

return ext
end

local function convert_to_ref(task, nonce, queue_id, part, external_refs)
  local path = string.format('/%s-%s-%s.%s', queue_id, nonce,
      rspamd_text.randombytes(8):base32(), gen_ext('raw'))
  local content = part.content

if settings.zstd_compress then
    external_refs[path] = rspamd_util.zstd_compress(content)
  else
    external_refs[path] = content
  end

part.content = nil
  part.content_path = path

return path
end

local function structured_data(task, nonce, queue_id)
  local content, content_type
  local external_refs = {}
  local lua_mime = require "lua_mime"
  local ucl = require "ucl"

local message_split = lua_mime.message_to_ucl(task)
  if settings.inline_content_limit and settings.inline_content_limit > 0 then

for i, part in ipairs(message_split.parts or {}) do
      if part.content and #part.content >= settings.inline_content_limit then
        local ref = convert_to_ref(task, nonce, queue_id, part, external_refs)
        lua_util.debugm(N, task, "convert part number %s to a reference %s",
            i, ref)
      end
    end
  end

if settings.zstd_compress then
    content = rspamd_util.zstd_compress(ucl.to_format(message_split, 'msgpack'))
    content_type = 'application/zstd'
  else
    content = ucl.to_format(message_split, 'msgpack')
    content_type = 'application/msgpack'
  end

local path = string.format('/%s-%s.%s', queue_id, nonce, gen_ext('msgpack'))

return path, content, content_type, external_refs
end

local function s3_aws_callback(task)
  local uri = string.format('https://%s.%s', settings.s3_bucket, settings.s3_host)
  -- Create a nonce
  local nonce = rspamd_text.randombytes(16):base32()
  local queue_id = task:get_queue_id()
  if not queue_id then
    queue_id = rspamd_text.randombytes(8):base32()
  end
  -- Hack to pass host
  local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host)

local function gen_s3_http_callback(path, what)
    return function(http_err, code, body, headers)

if http_err then
        if settings.fail_action then
          task:set_pre_result(settings.fail_action,
              string.format('S3 save failed: %s', http_err), N,
              nil, nil, 'least')
        end
        rspamd_logger.errx(task, 'cannot save %s to AWS S3: %s', path, http_err)
      else
        rspamd_logger.messagex(task, 'saved %s successfully in S3 object %s', what, path)
      end
      lua_util.debugm(N, task, 'obj=%s, err=%s, code=%s, body=%s, headers=%s',
          path, http_err, code, body, headers)
    end
  end

if settings.save_raw then
    local path, content, content_type = raw_data(task, nonce, queue_id)
    local hdrs = lua_aws.aws_request_enrich({
      region = settings.s3_region,
      headers = {
        ['Content-Type'] = content_type,
        ['Host'] = aws_host
      },
      uri = path,
      key_id = settings.s3_key_id,
      secret_key = settings.s3_secret_key,
      method = 'PUT',
    }, content)
    rspamd_http.request({
      url = uri .. path,
      task = task,
      method = 'PUT',
      body = content,
      callback = gen_s3_http_callback(path, 'raw message'),
      headers = hdrs,
      timeout = settings.s3_timeout,
    })
  end
  if settings.save_structure then
    local path, content, content_type, external_refs = structured_data(task, nonce, queue_id)
    local hdrs = lua_aws.aws_request_enrich({
      region = settings.s3_region,
      headers = {
        ['Content-Type'] = content_type,
        ['Host'] = aws_host
      },
      uri = path,
      key_id = settings.s3_key_id,
      secret_key = settings.s3_secret_key,
      method = 'PUT',
    }, content)
    rspamd_http.request({
      url = uri .. path,
      task = task,
      method = 'PUT',
      body = content,
      callback = gen_s3_http_callback(path, 'structured message'),
      headers = hdrs,
      upstream = settings.upstreams:get_upstream_round_robin(),
      timeout = settings.s3_timeout,
    })

for ref, part_content in pairs(external_refs) do
      local part_hdrs = lua_aws.aws_request_enrich({
        region = settings.s3_region,
        headers = {
          ['Content-Type'] = content_type,
          ['Host'] = aws_host
        },
        uri = ref,
        key_id = settings.s3_key_id,
        secret_key = settings.s3_secret_key,
        method = 'PUT',
      }, part_content)
      rspamd_http.request({
        url = uri .. ref,
        task = task,
        upstream = settings.upstreams:get_upstream_round_robin(),
        method = 'PUT',
        body = part_content,
        callback = gen_s3_http_callback(ref, 'part content'),
        headers = part_hdrs,
        timeout = settings.s3_timeout,
      })
    end
  end

end

local opts = rspamd_config:get_all_opt('aws_s3')
if not opts then
  return
end

settings = lua_util.override_defaults(settings, opts)
local res, err = settings_schema:transform(settings)

if not res then
  rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err)
  lua_util.disable_module(N, "config")
  return
end

rspamd_logger.infox(rspamd_config, 'enabled AWS s3 dump to %s', res.s3_bucket)

settings = res

settings.upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(),
    string.format('https://%s.%s', settings.s3_bucket, settings.s3_host))

if not settings.upstreams then
  rspamd_logger.warnx(rspamd_config, 'cannot parse hostname: %s',
      string.format('https://%s.%s', settings.s3_bucket, settings.s3_host))
  lua_util.disable_module(N, "config")
  return
end

local is_postfilter = settings.fail_action ~= nil

rspamd_config:register_symbol({
  name = 'EXPORT_AWS_S3',
  type = is_postfilter and 'postfilter' or 'idempotent',
  callback = s3_aws_callback,
  augmentations = { string.format("timeout=%f", settings.s3_timeout) },
  priority = is_postfilter and lua_util.symbols_priorities.high or nil,
  flags = 'empty,explicit_disable,ignore_passthrough,nostat',
})

${this.title}

Code Editor : aws_s3.lua