数据导出微服务设计

文章目录[隐藏]
WPJAM TOC
数据导出基础服务>

需求分析

  • 为企业多个服务提供数据导出中转处理服务

  • 解决导出数据过大,导出数据时间过长,频繁导出相同数据资源浪费等问题

  • 解决各个应用导出数据存放不统一,导出浪费时间资源的情况

  • 初期只作为一个队列中转,统一处理储存,数据去重

请求必须

  1. 获取数据源地址:端口+请求参数+请求方式

  2. 源地址导出数据源的大小和页数

  3. 回调地址

架构流程图

数据表结构

  • 数据表sql

  • CREATE TABLE export_files
    (
        id bigint(20) PRIMARY KEY AUTO_INCREMENT,
        hash_key char(32) DEFAULT '' NOT NULL COMMENT '请求哈希md5唯一标识',
        oss_object varchar(255) DEFAULT '' NOT NULL COMMENT 'oss文件对象',
        created_at timestamp,
        updated_at timestamp
    );
    CREATE UNIQUE INDEX export_files_hash_key_uindex ON export_files (hash_key);
    ALTER TABLE export_files COMMENT = '导出文件数据表';
    
    
    CREATE TABLE export_logs
    (
        id bigint(20) PRIMARY KEY AUTO_INCREMENT,
        c text COMMENT '请求参数',
        status tinyint(2) DEFAULT 0 NOT NULL COMMENT '状态:0=默认(处理中);1=导出成功;2=导出失败',
        retry_num tinyint(4) DEFAULT 0 NOT NULL COMMENT '重试次数',
        callback_url varchar(255) DEFAULT '' NOT NULL COMMENT '回调地址',
        return_data text COMMENT '返回数据体(成功或失败)',
        app_id varchar(32) DEFAULT '' NOT NULL COMMENT '请求客户端渠道',
        hash_key char(32) DEFAULT '' NOT NULL COMMENT '请求基本参数的hash_key(关联export_file.hash_key)',
        created_at timestamp,
        updated_at timestamp
    );
    ALTER TABLE export_logs COMMENT = '导出日志表';
    
            $parameter = [
              //请求时间戳
                'timestamp' => time(),    
              //获取数据源类型:HTTP/DB(目前只支持http)
                'request_type' => 'HTTP', 
              //获取数据源请求方法:推荐使用post,因为有些参数里面包含对个数据,
                'request_method' => 'GET',
              //数据源请求接口路由地址(针对http获取数据源方式),
                'request_url' => 'http://testv21.onlineadmin.qsyuwen.cn/api/finance/order/list',
              //请求数据源携带请求参数示例:
                'request_parameters' => [
                    'status' => 1,
                    'pay_type_id' => 3,
                    'dateTime' => [
                        '2019-05-09 21:00:00',
                        '2019-05-23 17:00:00'
                    ],
                ],
              //回调路由地址
                'callback_url' => 'http://xthk.com/export/callback',
            ];
     
            $parameter = [
                'timestamp' => time(),
                'request_type' => 'HTTP',
                'request_method' => 'GET',
                'request_url' => 'http://testv21.onlineadmin.qsyuwen.cn/api/finance/order/list',
                'request_parameters' => [
                    'status' => 1,
                    'pay_type_id' => 3,
                    'dateTime' => [
                        '2019-05-09 21:00:00',
                        '2019-05-23 17:00:00'
                    ],
                ],
                'callback_url' => 'http://xthk.com/export/callback',
            ];
            ksort($parameter);
            $key = md5('你的app_secret');
            $crypt = new \Illuminate\Encryption\Encrypter($key, 'AES-256-CBC');
            $pload = $crypt->encrypt($parameter);    //加密
            dump($pload);
            /*
             * 加密结果
             * eyJpdiI6Im1uK1BtQW50SUEwM3pSTkNwVmFiXC9BPT0iLCJ2YWx1ZSI6IjRycXp5OEZoZHdRUmtwRUlaY2t6V3hIVStSTTcyTzFtckVmNmk0YlNlWitnTW1UK1JNcU1QbEtXWHVBMGhYZmZrbkFCK093OWF1WFR6V0xxVG5aWTVGcGNTOEVXbWJpWnJINkd0TXFlVnlnWHVHNklBMmw3NjhONkw1alB1OUNpNWl2Ym13U3lMXC9MNHBQMUlhSTdERWwwNWFKZmFUNlpaenVjdXFKNUFSeUNocXUyd3QrXC9Fb3BRbVBaU3g5V0U3UG9NRjlMSHREdDJXcjlRSVB0TlJcLzZzSUpcL3E4ZTJcLzF0T1RXcWt5bWFLKzVyenFwN0J2VnlxdUp6MmYxQjBPb0hrU0hNXC94YUZVdE1nejZCeGlYbXBIcWxPMWtlYnJkUlRxWWZycktZOTBOcUdHMmFFajY0ZjJwclh0cTRZSnlHdUs2NGQrTDhCNGs2cW54bUc2QVgydSt1SjFIejJ4RjhhcWdlTHp6VDFicG8rY1k2OHpoK2ZpcTdnM05SUGM1RUl4R3ZmQjFjZnZXN29nWmxYNGxiXC9cL1FVM2VwUTVUU3dmUlhcLzY4a1MxYWVYMlBiK2tuTnZvVEtrYUFPYm5JM012dzA5SjdLZlE3VUhMK214ckM2em5mZTRoMUhSM3dcL011d1wvakc4b3pMcnJoWVB4c2l2VDV2SnBYV1VXcVpaQTJGMU1KVlZTXC8wclcxQ3Y3eVlUXC8xYkE9PSIsIm1hYyI6ImMwNGY0MzMxZjc4MGFkNmJmOGI4ZGQ0OGQxNzRkNjE5Mjg1NWFhNjc3MzQ5NWQ1MmRjMTlkMmM2NjlhODExYzgifQ==
             */
    
            $value = $crypt->decrypt($pload);   //解密
            dump($value);
    //  解密结果
    //       [
    //            "callback_url" => "http://xthk.com/export/callback",
    //            "request_method" => "GET",
    //            "request_parameters" => [
    //                "status" => 1,
    //                "pay_type_id" => 3,
    //                "dateTime" => [
    //                    "2019-05-09 21:00:00",
    //                    "2019-05-23 17:00:00",
    //                ],
    //            ],
    //            "request_type" => "HTTP",
    //            "request_url" => "http://testv21.onlineadmin.qsyuwen.cn/api/finance/order/list",
    //            "timestamp" => 1558680609
    //        ];
    
    <?php
    
    namespace Illuminate\Encryption;
    
    use RuntimeException;
    use Illuminate\Contracts\Encryption\DecryptException;
    use Illuminate\Contracts\Encryption\EncryptException;
    use Illuminate\Contracts\Encryption\Encrypter as EncrypterContract;
    
    class Encrypter implements EncrypterContract
    {
        /**
         * The encryption key.
         *
         * @var string
         */
        protected $key;
    
        /**
         * The algorithm used for encryption.
         *
         * @var string
         */
        protected $cipher;
    
        /**
         * Create a new encrypter instance.
         *
         * @param  string  $key
         * @param  string  $cipher
         * @return void
         *
         * @throws \RuntimeException
         */
        public function __construct($key, $cipher = 'AES-128-CBC')
        {
            $key = (string) $key;
    
            if (static::supported($key, $cipher)) {
                $this->key = $key;
                $this->cipher = $cipher;
            } else {
                throw new RuntimeException('The only supported ciphers are AES-128-CBC and AES-256-CBC with the correct key lengths.');
            }
        }
    
        /**
         * Determine if the given key and cipher combination is valid.
         *
         * @param  string  $key
         * @param  string  $cipher
         * @return bool
         */
        public static function supported($key, $cipher)
        {
            $length = mb_strlen($key, '8bit');
    
            return ($cipher === 'AES-128-CBC' && $length === 16) ||
                   ($cipher === 'AES-256-CBC' && $length === 32);
        }
    
        /**
         * Create a new encryption key for the given cipher.
         *
         * @param  string  $cipher
         * @return string
         */
        public static function generateKey($cipher)
        {
            return random_bytes($cipher === 'AES-128-CBC' ? 16 : 32);
        }
    
        /**
         * Encrypt the given value.
         *
         * @param  mixed  $value
         * @param  bool  $serialize
         * @return string
         *
         * @throws \Illuminate\Contracts\Encryption\EncryptException
         */
        public function encrypt($value, $serialize = true)
        {
            $iv = random_bytes(openssl_cipher_iv_length($this->cipher));
    
            // First we will encrypt the value using OpenSSL. After this is encrypted we
            // will proceed to calculating a MAC for the encrypted value so that this
            // value can be verified later as not having been changed by the users.
            $value = \openssl_encrypt(
                $serialize ? serialize($value) : $value,
                $this->cipher, $this->key, 0, $iv
            );
    
            if ($value === false) {
                throw new EncryptException('Could not encrypt the data.');
            }
    
            // Once we get the encrypted value we'll go ahead and base64_encode the input
            // vector and create the MAC for the encrypted value so we can then verify
            // its authenticity. Then, we'll JSON the data into the "payload" array.
            $mac = $this->hash($iv = base64_encode($iv), $value);
    
            $json = json_encode(compact('iv', 'value', 'mac'));
    
            if (json_last_error() !== JSON_ERROR_NONE) {
                throw new EncryptException('Could not encrypt the data.');
            }
    
            return base64_encode($json);
        }
    
        /**
         * Encrypt a string without serialization.
         *
         * @param  string  $value
         * @return string
         *
         * @throws \Illuminate\Contracts\Encryption\EncryptException
         */
        public function encryptString($value)
        {
            return $this->encrypt($value, false);
        }
    
        /**
         * Decrypt the given value.
         *
         * @param  mixed  $payload
         * @param  bool  $unserialize
         * @return mixed
         *
         * @throws \Illuminate\Contracts\Encryption\DecryptException
         */
        public function decrypt($payload, $unserialize = true)
        {
            $payload = $this->getJsonPayload($payload);
    
            $iv = base64_decode($payload['iv']);
    
            // Here we will decrypt the value. If we are able to successfully decrypt it
            // we will then unserialize it and return it out to the caller. If we are
            // unable to decrypt this value we will throw out an exception message.
            $decrypted = \openssl_decrypt(
                $payload['value'], $this->cipher, $this->key, 0, $iv
            );
    
            if ($decrypted === false) {
                throw new DecryptException('Could not decrypt the data.');
            }
    
            return $unserialize ? unserialize($decrypted) : $decrypted;
        }
    
        /**
         * Decrypt the given string without unserialization.
         *
         * @param  string  $payload
         * @return string
         *
         * @throws \Illuminate\Contracts\Encryption\DecryptException
         */
        public function decryptString($payload)
        {
            return $this->decrypt($payload, false);
        }
    
        /**
         * Create a MAC for the given value.
         *
         * @param  string  $iv
         * @param  mixed  $value
         * @return string
         */
        protected function hash($iv, $value)
        {
            return hash_hmac('sha256', $iv.$value, $this->key);
        }
    
        /**
         * Get the JSON array from the given payload.
         *
         * @param  string  $payload
         * @return array
         *
         * @throws \Illuminate\Contracts\Encryption\DecryptException
         */
        protected function getJsonPayload($payload)
        {
            $payload = json_decode(base64_decode($payload), true);
    
            // If the payload is not valid JSON or does not have the proper keys set we will
            // assume it is invalid and bail out of the routine since we will not be able
            // to decrypt the given value. We'll also check the MAC for this encryption.
            if (! $this->validPayload($payload)) {
                throw new DecryptException('The payload is invalid.');
            }
    
            if (! $this->validMac($payload)) {
                throw new DecryptException('The MAC is invalid.');
            }
    
            return $payload;
        }
    
        /**
         * Verify that the encryption payload is valid.
         *
         * @param  mixed  $payload
         * @return bool
         */
        protected function validPayload($payload)
        {
            return is_array($payload) && isset($payload['iv'], $payload['value'], $payload['mac']) &&
                   strlen(base64_decode($payload['iv'], true)) === openssl_cipher_iv_length($this->cipher);
        }
    
        /**
         * Determine if the MAC for the given payload is valid.
         *
         * @param  array  $payload
         * @return bool
         */
        protected function validMac(array $payload)
        {
            $calculated = $this->calculateMac($payload, $bytes = random_bytes(16));
    
            return hash_equals(
                hash_hmac('sha256', $payload['mac'], $bytes, true), $calculated
            );
        }
    
        /**
         * Calculate the hash of the given payload.
         *
         * @param  array  $payload
         * @param  string  $bytes
         * @return string
         */
        protected function calculateMac($payload, $bytes)
        {
            return hash_hmac(
                'sha256', $this->hash($payload['iv'], $payload['value']), $bytes, true
            );
        }
    
        /**
         * Get the encryption key.
         *
         * @return string
         */
        public function getKey()
        {
            return $this->key;
        }
    }