我们在开发项目的时候最能体现能力的地方是出了问题能快速定位和解决问题,如果按照框架原有的使用文件记录日志比较麻烦的地方就是需要登录服务器执行linux命令过滤,如果日志量比较多可能日志不会保持太久,一般会加日志定期清理的linux脚本,如果能记录到数据库中,并且可以按照业务进行日志查询,还可以自己决定多久清理一次,这些都是程序员比较擅长的,而且默认的框架是不记录响应内容、执行时长等细节的。
优化后的项目有以下特点:
采用Laravel的数据库ORM
当然,你也可以直接拉到最底下跳到成果~
php>=7.3
redis
mysql
直接使用composer命令安装全新项目
composer create-project workerman/webman
安装redis异步队列、env环境包、guzzlehttp
composer install webman/redis-queue vlucas/phpdotenv guzzlehttp/guzzle
安装laravel的ORM
composer require -W illuminate/database illuminate/pagination illuminate/events symfony/var-dumper
在项目根目录创建.env文件
APP_DEBUG=true
DB_DRIVER=mysql
DB_HOST=localhost
DB_PORT=3306
DB_DATABASE=aaa
DB_USERNAME=root
DB_PASSWORD=root
DB_CHARSET=utf8mb4
DB_COLLATION=utf8mb4_unicode_ci
DB_PREFIX=
REDIS_HOST=127.0.0.1
REDIS_PORT=6379
REDIS_AUTH=admin
REDIS_DB=0
REDIS_QUEUE_HOST=127.0.0.1
REDIS_QUEUE_PORT=6379
REDIS_QUEUE_AUTH=admin
REDIS_QUEUE_DB=0
#加解密key
ENCRYPT_KEY=
在app\functions.php中增加两个方法,这里统一接口的返回值结构
function apiSuccess(array $data = [])
{
$res = [
'code' => \app\enum\ErrorCode::SUCCESS,
'msg' => 'ok',
];
if ($data) {
$res['data'] = $data;
}
return $res;
}
function apiError(int $code, string $msg, array $data = [], array $trace = [])
{
$res = [
'code' => $code,
'msg' => $msg,
];
if ($data) {
$res['data'] = $data;
}
if ($trace) {
$res['trace'] = $trace;
}
return $res;
}
编辑support\helpers.php在结尾增加如下代码(这两个方法是从hyperf框架copy过来的):
if (! function_exists('env')) {
/**
* Gets the value of an environment variable.
*
* @param string $key
* @param null|mixed $default
*/
function env(string $key, $default = null)
{
$value = getenv($key);
if ($value === false) {
return value($default);
}
switch (strtolower($value)) {
case 'true':
case '(true)':
return true;
case 'false':
case '(false)':
return false;
case 'empty':
case '(empty)':
return '';
case 'null':
case '(null)':
return null;
}
if (($valueLength = strlen($value)) > 1 && $value[0] === '"' && $value[$valueLength - 1] === '"') {
return substr($value, 1, -1);
}
return $value;
}
}
if (! function_exists('value')) {
/**
* Return the default value of the given value.
*
* @param mixed $value
*/
function value($value)
{
return $value instanceof \Closure ? $value() : $value;
}
}
创建文件 app\enum\ErrorCode.php
<?php
namespace app\enum;
class ErrorCode
{
public const SUCCESS = 200;
public const SERVER_ERROR = 500;
//加密错误
public const ENCRYPTER_ERROR = 3000;
//解密错误
public const EDCRYPTER_ERROR = 3001;
}
config\plugin\webman\redis-queue\redis.php
<?php
return [
'default' => [
'host' => 'redis://' . env('REDIS_QUEUE_HOST', '127.0.0.1') . ':' . env('REDIS_QUEUE_PORT', 6379),
'options' => [
'auth' => env('REDIS_QUEUE_AUTH', null), // 密码,字符串类型,可选参数
'db' => env('REDIS_QUEUE_DB', 0), // 数据库
'prefix' => '', // key 前缀
'max_attempts' => 3, // 消费失败后,重试次数
'retry_seconds' => 5, // 重试间隔,单位秒
]
],
];
config\app.php,主要是把写死的debug改成了env获取
'debug' => env('APP_DEBUG', false),
当开启debug时,如果服务发生异常,接口返回值会增加trace字段,帮助调试问题:
config\database.php
<?php
return [
// 默认数据库
'default' => 'mysql',
// 各种数据库配置
'connections' => [
'mysql' => [
'driver' => 'mysql',
'host' => env('DB_HOST', 'localhost'),
'port' => env('DB_PORT', 3306),
'database' => env('DB_DATABASE', 'test'),
'username' => env('DB_USERNAME', 'root'),
'password' => env('DB_PASSWORD', 'root'),
'unix_socket' => '',
'charset' => env('DB_CHARSET', 'utf8mb4'),
'collation' => env('DB_COLLATION', 'utf8mb4_unicode_ci'),
'prefix' => env('DB_PREFIX', ''),
'strict' => true,
'engine' => null,
],
],
];
增加 config\http.php,这里是guzzlehttp的配置,自己按需添加
<?php
return [
'timeout' => 5.0
];
config\redis.php
<?php
return [
'default' => [
'host' => env('REDIS_HOST', '127.0.0.1'),
'password' => env('REDIS_QUEUE_AUTH', null),
'port' => env('REDIS_PORT,6379'),
'database' => env('REDIS_DB', 0),
],
];
新建中间件 app/middleware/GlobalLog.php,框架的访问和响应日志在这个中间件中记录。
<?php
namespace app\middleware;
use app\enum\ErrorCode;
use support\Context;
use support\Log;
use Webman\MiddlewareInterface;
use Webman\Http\Response;
use Webman\Http\Request;
use Webman\RedisQueue\Redis;
/**
* Class GlobalLog
* @package app\middleware
*/
class GlobalLog implements MiddlewareInterface
{
public function process(Request $request, callable $next): Response
{
$start = microtime(true);
//获取请求信息
$data = [
'ip' => $this->getIp($request),
'uri' => $request->uri(),
'method' => $request->method(),
'appid' => '', //TODO 业务数据,如果项目中可直接获取到appid,记录在此处
'traceid' => $request->header('traceid', md5(microtime())),
'refer' => $request->header('referer'),
'user_agent' => $request->header('user-agent'),
'query' => $request->all(),
'cookie' => $request->cookie(),
'created_at' => date('Y-m-d H:i:s'),
];
//记录全局traceid
Context::set('traceid', $data['traceid']);
/** @var Response $response */
$response = $next($request);
$err = $response->exception();
$res = [];
if ($err instanceof \Exception) {
//这个统一异常时的接口响应
$trace = [$err->getMessage(), $err->getFile(), $err->getLine(), $err->getTraceAsString()];
$data['exception'] = json_encode($trace, JSON_UNESCAPED_UNICODE);
Log::error('server error', $trace);
if (env('APP_DEBUG')) {
$res = apiError(ErrorCode::SERVER_ERROR, '服务异常,请稍后重试', [], $trace);
} else {
$res = apiError(ErrorCode::SERVER_ERROR, '服务异常,请稍后重试');
}
}
$data['errcode'] = $response->getStatusCode();
$data['response'] = $res ? json_encode($res, JSON_UNESCAPED_UNICODE) : $response->rawBody();
$end = microtime(true);
$exec_time = round(($end - $start) * 1000, 2);
$data['exec_time'] = $exec_time;
//投递到异步队列
Redis::send('global-log', $data);
if ($res) {
return json($res);
}
return $response;
}
private function getIp(Request $request)
{
$forward_ip = $request->header('X-Forwarded-For');
$ip1 = $request->header('x-real-ip');
$ip2 = $request->header('remote_addr');
if (!$ip1 && !$ip2 && !$forward_ip) {
return false;
}
$request_ips = [];
if ($forward_ip) {
$request_ips[] = $forward_ip;
}
if ($ip1) {
$request_ips[] = $ip1;
}
if ($ip2) {
$request_ips[] = $ip2;
}
return implode(',', $request_ips);
}
}
增加 app\queue\redis\GlobalLog.php
如果需要调整按天分表的策略或者调整日志字段可在该文件中调整。
<?php
namespace app\queue\redis;
use app\service\EncrypterService;
use Illuminate\Database\Schema\Blueprint;
use support\Db;
use support\Log;
use Webman\RedisQueue\Consumer;
class GlobalLog implements Consumer
{
// 要消费的队列名
public $queue = 'global-log';
// 连接名,对应 plugin/webman/redis-queue/redis.php 里的连接`
public $connection = 'default';
// 消费
public function consume($data)
{
try {
$tableName = 'global_log_' . date('Ymd');
$this->initTable($tableName);
$cookie = $data['cookie'] ?? [];
$appid = $data['appid'] ?? '';
$query = $data['query'] ?? [];
$ticket = $query['ticket'] ?? '';
if (!$appid && $ticket) {
//该部分为业务的数据处理,可根据业务进行调整,不需要可删除
//从ticket中解析appid
try {
$ticketStr = (new EncrypterService())->decrypt($ticket);
$ticketArr = explode('|', $ticketStr);
$appid = $ticketArr[0] ?? '';
} catch (\Exception $e) {
//ticket解密失败,可能是环境不匹配或者ticket不正确,不处理
$appid = 'ticket_error';
}
}
DB::table($tableName)->insert([
'ip' => $data['ip'] ?? '',
'uri' => $data['uri'] ?? '',
'method' => $data['method'] ?? '',
'appid' => $appid,
'traceid' => $data['traceid'] ?? '',
'referer' => $data['referer'] ?? '',
'user_agent' => $data['user_agent'] ?? '',
'query' => $query ? json_encode($query, JSON_UNESCAPED_UNICODE) : '',
'errcode' => $data['errcode'] ?? '',
'response' => $data['response'] ?? '',
'exception' => $data['exception'] ?? '',
'exec_time' => $data['exec_time'] ?? '',
'cookie' => $cookie ? json_encode($data['cookie'], JSON_UNESCAPED_UNICODE) : '',
'created_at' => date('Y-m-d H:i:s'),
]);
} catch (\Throwable $e) {
Log::error('global_log_queue_error', [
'msg' => $e->getMessage(),
'line' => $e->getLine(),
'file' => $e->getFile(),
'trace' => $e->getTraceAsString()
]);
}
}
private function initTable($tableName)
{
//判断global_log表是否存在,按天分表
if (!Db::schema()->hasTable($tableName)) {
Db::schema()->create($tableName, function (Blueprint $table) {
$table->increments('id')->autoIncrement()->unsigned();
$table->string('ip', 20)->nullable(true)->default(null)->comment('访问ip');
$table->string('uri', 255)->nullable(true)->default(null)->comment('访问uri');
$table->string('method', 10)->nullable(true)->default(null)->comment('get or post');
$table->string('appid', 50)->nullable(true)->default(null)->comment('应用平台appid');
$table->string('traceid', 255)->nullable(true)->default(null)->comment('traceid');
$table->text('referer')->nullable(true)->default(null)->comment('来源页');
$table->text('user_agent')->nullable(true)->default(null)->comment('user_agent');
$table->text('query')->nullable(true)->default(null)->comment('请求参数');
$table->string('errcode', 10)->nullable(true)->default(null)->comment('响应错误码');
$table->text('response')->nullable(true)->default(null)->comment('响应结果');
$table->text('exception')->nullable(true)->default(null)->comment('异常信息');
$table->text('exec_time')->nullable(true)->default(null)->comment('执行时间,单位毫秒');
$table->text('cookie')->nullable(true)->default(null)->comment('请求cookie');
$table->dateTime('created_at')->nullable(true)->default(null);
$table->index('ip', 'ip');
$table->index('uri', 'uri');
$table->index('appid', 'appid');
$table->index('traceid', 'traceid');
$table->index('created_at', 'created_at');
$table->charset = 'utf8mb4';
$table->collation = 'utf8mb4_unicode_ci';
$table->engine = 'InnoDB';
});
}
}
}
配置文件 app\config\middleware.php
注意:如果项目中有多个全局中间件,这个中间件一定要放在第一个。
<?php
return [
//全局中间件
'' => [
app\middleware\GlobalLog::class
]
];
增加 support\Http.php
<?php
declare(strict_types=1);
namespace support;
use GuzzleHttp\Client;
class Http
{
/**
* @var \GuzzleHttp\Client
*/
private $httpClient;
public function __construct()
{
$this->createHttpClient();
}
/**
* get请求
* @param string $url
* @param array $query
* @return mixed
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* @throws \Throwable
*/
public function get(string $url, array $query = [])
{
$data = [
'query' => $query,
];
return $this->exec('GET', $url, $data);
}
/**
* post请求
* @param string $url
* @param array $params post form参数
* @param array $headers 请求头
* @param bool $json 参数是否是json格式
* @return false|mixed
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* @throws \Throwable
*/
public function post(string $url, array $params = [], array $headers = [], bool $json = false)
{
$data = ['headers' => $headers];
if ($json) {
$data['json'] = $params;
} else {
$data['form_params'] = $params;
}
return $this->exec('POST', $url, $data);
}
/**
* @param $method
* @param $url
* @param $data
* @return string
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* @throws \Throwable
*/
private function exec($method, $url, $data): string
{
$exception = null;
$start = microtime(true);
try {
$response = $this->httpClient->request($method, $url, $data);
$result = $response->getBody()->getContents();
} catch (\Throwable $e) {
$exception = $e;
}
$end = microtime(true);
$exec_time = round(($end - $start) * 1000, 2);
$log = [
'url' => $url,
'method' => $method,
'data' => $data,
'trace_id' => Context::get('traceid', ''),
'exception' => $exception ? [$exception->getMessage(), $exception->getFile(), $exception->getLine(), $exception->getTraceAsString()] : [],
'response' => $result ?? '',
'exec_time' => $exec_time,
];
\Webman\RedisQueue\Redis::send('http-log', $log);
if ($exception instanceof \Throwable) {
Log::error('http request error:' . $exception->getMessage(), $log);
throw $exception;
}
return $result ?? '';
}
private function createHttpClient()
{
if (!$this->httpClient instanceof Client) {
$this->httpClient = new Client([
'timeout' => config('http.timeout', 5),
]);
}
}
}
增加 app\queue\redis\HttpLog.php
<?php
namespace app\queue\redis;
use Illuminate\Database\Schema\Blueprint;
use support\Db;
use support\Log;
use Webman\RedisQueue\Consumer;
class HttplLog implements Consumer
{
// 要消费的队列名
public $queue = 'http-log';
// 连接名,对应 plugin/webman/redis-queue/redis.php 里的连接`
public $connection = 'default';
// 消费
public function consume($data)
{
try {
$tableName = 'http_log_' . date('Ymd');
$this->initTable($tableName);
$request_data = $data['data'] ?? [];
$exception = $data['exception'] ?? [];
Db::table($tableName)->insert([
'url' => $data['url'] ?? '',
'method' => $data['method'] ?? '',
'data' => $request_data ? json_encode($request_data, JSON_UNESCAPED_UNICODE) : '',
'trace_id' => $data['trace_id'] ?? '',
'exception' => $exception ? json_encode($exception, JSON_UNESCAPED_UNICODE) : '',
'response' => $data['response'] ?? '',
'exec_time' => $data['exec_time'] ?? '',
'created_at' => date('Y-m-d H:i:s'),
]);
} catch (\Throwable $e) {
Log::error('http log error', [
'msg' => $e->getMessage(),
'line' => $e->getLine(),
'file' => $e->getFile(),
'trace' => $e->getTraceAsString()
]);
}
}
private function initTable($tableName)
{
//判断global_log表是否存在,按天分表
if (!Db::schema()->hasTable($tableName)) {
Db::schema()->create($tableName, function (Blueprint $table) {
$table->increments('id')->autoIncrement()->unsigned();
$table->string('url', 255)->nullable(true)->default(null)->comment('访问url');
$table->string('method', 10)->nullable(true)->default(null)->comment('get or post');
$table->text('data')->nullable(true)->comment('请求参数');
$table->string('trace_id', 200)->nullable(true)->comment('trace_id');
$table->text('exception')->nullable(true)->comment('异常信息');
$table->text('response')->nullable(true)->comment('响应结果');
$table->string('exec_time', 10)->nullable(true)->comment('执行时长,单位毫秒');
$table->dateTime('created_at')->nullable(true)->default(null);
$table->index('trace_id', 'idx_trace_id');
$table->index('created_at', 'idx_created_at');
$table->charset = 'utf8mb4';
$table->collation = 'utf8mb4_unicode_ci';
$table->engine = 'InnoDB';
});
}
}
}
<?php
declare(strict_types=1);
namespace support;
use app\enum\ErrorCode;
class Encrypter
{
/**
* The encryption key.
*
* @var string
*/
protected $key;
/**
* The algorithm used for encryption.
*
* @var string
*/
protected $cipher;
/**
* Create a new encrypter instance.
*
* @param string $key
* @param string $cipher
* @return void
*
* @throws \Exception
*/
public function __construct($key, $cipher = 'AES-256-CBC')
{
$key = (string)$key;
if ($this->supported($key, $cipher)) {
$this->key = $key;
$this->cipher = $cipher;
} else {
throw new \Exception('The only supported ciphers are AES-128-CBC and AES-256-CBC with the correct key lengths.', ErrorCode::ENCRYPTER_ERROR);
}
}
/**
* Determine if the given key and cipher combination is valid.
*
* @param string $key
* @param string $cipher
* @return bool
*/
public function supported($key, $cipher)
{
$length = mb_strlen($key, '8bit');
return ($cipher === 'AES-128-CBC' && $length === 16) ||
($cipher === 'AES-256-CBC' && $length === 32);
}
/**
* Create a new encryption key for the given cipher.
*
* @param string $cipher
* @return string
*/
public function generateKey($cipher)
{
return random_bytes($cipher === 'AES-128-CBC' ? 16 : 32);
}
/**
* Encrypt the given value.
*
* @param mixed $value
* @param bool $serialize
* @return string
*
* @throws \Exception
*/
public function encrypt($value, $serialize = true)
{
$iv = random_bytes(openssl_cipher_iv_length($this->cipher));
// First we will encrypt the value using OpenSSL. After this is encrypted we
// will proceed to calculating a MAC for the encrypted value so that this
// value can be verified later as not having been changed by the users.
$value = \openssl_encrypt(
$serialize ? serialize($value) : $value,
$this->cipher, $this->key, 0, $iv
);
if ($value === false) {
throw new \Exception('Could not encrypt the data.', ErrorCode::ENCRYPTER_ERROR);
}
// Once we get the encrypted value we'll go ahead and base64_encode the input
// vector and create the MAC for the encrypted value so we can then verify
// its authenticity. Then, we'll JSON the data into the "payload" array.
$mac = $this->hash($iv = base64_encode($iv), $value);
$json = json_encode(compact('iv', 'value', 'mac'));
if (json_last_error() !== JSON_ERROR_NONE) {
throw new \Exception('Could not encrypt the data.', ErrorCode::ENCRYPTER_ERROR);
}
return base64_encode($json);
}
/**
* Encrypt a string without serialization.
*
* @param string $value
* @return string
*
* @throws \Illuminate\Contracts\Encryption\EncryptException
*/
public function encryptString($value)
{
return $this->encrypt($value, false);
}
/**
* Decrypt the given value.
*
* @param string $payload
* @param bool $unserialize
* @return mixed
*
* @throws \Exception
*/
public function decrypt($payload, $unserialize = true)
{
$payload = $this->getJsonPayload($payload);
$iv = base64_decode($payload['iv']);
// Here we will decrypt the value. If we are able to successfully decrypt it
// we will then unserialize it and return it out to the caller. If we are
// unable to decrypt this value we will throw out an exception message.
$decrypted = \openssl_decrypt(
$payload['value'], $this->cipher, $this->key, 0, $iv
);
if ($decrypted === false) {
throw new \Exception('Could not decrypt the data.', ErrorCode::ENCRYPTER_ERROR);
}
return $unserialize ? unserialize($decrypted) : $decrypted;
}
/**
* Decrypt the given string without unserialization.
*
* @param string $payload
* @return string
*
* @throws \Exception
*/
public function decryptString($payload)
{
return $this->decrypt($payload, false);
}
/**
* Create a MAC for the given value.
*
* @param string $iv
* @param mixed $value
* @return string
*/
protected function hash($iv, $value)
{
return hash_hmac('sha256', $iv . $value, $this->key);
}
/**
* Get the JSON array from the given payload.
*
* @param string $payload
* @return array
*
* @throws \Exception
*/
protected function getJsonPayload($payload)
{
$payload = json_decode(base64_decode($payload), true);
// If the payload is not valid JSON or does not have the proper keys set we will
// assume it is invalid and bail out of the routine since we will not be able
// to decrypt the given value. We'll also check the MAC for this encryption.
if (!$this->validPayload($payload)) {
throw new \Exception('The payload is invalid.', ErrorCode::ENCRYPTER_ERROR);
}
if (!$this->validMac($payload)) {
throw new \Exception('The MAC is invalid.', ErrorCode::ENCRYPTER_ERROR);
}
return $payload;
}
/**
* Verify that the encryption payload is valid.
*
* @param mixed $payload
* @return bool
*/
protected function validPayload($payload)
{
return is_array($payload) && isset($payload['iv'], $payload['value'], $payload['mac']) &&
strlen(base64_decode($payload['iv'], true)) === openssl_cipher_iv_length($this->cipher);
}
/**
* Determine if the MAC for the given payload is valid.
*
* @param array $payload
* @return bool
*/
protected function validMac(array $payload)
{
$calculated = $this->calculateMac($payload, $bytes = random_bytes(16));
return hash_equals(
hash_hmac('sha256', $payload['mac'], $bytes, true), $calculated
);
}
/**
* Calculate the hash of the given payload.
*
* @param array $payload
* @param string $bytes
* @return string
*/
protected function calculateMac($payload, $bytes)
{
return hash_hmac(
'sha256', $this->hash($payload['iv'], $payload['value']), $bytes, true
);
}
/**
* Get the encryption key.
*
* @return string
*/
public function getKey()
{
return $this->key;
}
}
增加 app\service\EncrypterService.php
<?php
/**
* 加解密服务类(兼容laravel)
*/
namespace app\service;
use support\Encrypter;
class EncrypterService
{
protected $encrypter;
public function __construct()
{
$key = env('ENCRYPT_KEY');
if (!$this->encrypter instanceof Encrypter) {
$this->encrypter = new Encrypter($key);
}
}
public function encrypt($str)
{
return $this->encrypter->encrypt($str);
}
public function decrypt($str)
{
return $this->encrypter->decrypt($str);
}
}
到此框架已经修改完毕,如果看完后不想自己从头来一遍,可以把成品直接拿去用:
make
不容易,手动点赞!
牛牛,实用
点赞
太强了。爱了爱了,手动抄作业的感觉太好好
赞一个
make
点个赞
不错,赞。
最好的方法是接入openobserve 加入trace 和log。 日志压缩比高