"repositories": [ { "type": "path", "url": "D:\\projects\\imi" } ],
laravel 接入配置中心实现
/** * 配置信息合并 * * @param $arr1 * @param $arr2 * @return array */ if (!function_exists('array_recursive_merge')) { function array_recursive_merge($arr1, $arr2) { if(is_array($arr1) && is_array($arr2)) { foreach ($arr2 as $key => $item) { $arr1[$key] = array_recursive_merge($arr1[$key] ?? null, $item); } return $arr1; } return $arr2; } } //绑定配置文件加载事件 $app->afterBootstrapping(\Illuminate\Foundation\Bootstrap\LoadConfiguration::class, function($app){ //检查是否存在config cache if($app->configurationIsCached()) { return; } $config = $app['config']; $url = $app['config']['config']['uri'] ?? null; if($url) { $appName = strtolower($app['config']['app']['name']); $env = strtolower($app['config']['app']['env']); $url .= $appName . '-' . $env . '.yml'; $content = file_get_contents($url); if($content) { $content = \Symfony\Component\Yaml\Yaml::parse($content); foreach($content as $key=>$value) { $config[$key] = array_recursive_merge($config[$key]??null, $value); } } } //cache if($config['config']['cache'] ?? false) { $configPath = $app->getCachedConfigPath(); (new Illuminate\Filesystem\Filesystem)->put( $configPath, '<?php return '.var_export($config->all(), true).';'.PHP_EOL ); } });
golang 实现struct复制
package components
import "fmt"
import "reflect"
//复制结构体中名称一致类型一致得字段
func CopyFields(des interface{}, src interface{}, fields ...string) (err error) {
at := reflect.TypeOf(des)
av := reflect.ValueOf(des)
bt := reflect.TypeOf(src)
bv := reflect.ValueOf(src)
// 简单判断下
if at.Kind() != reflect.Ptr {
err = fmt.Errorf("des must be a struct pointer")
return
}
av = reflect.ValueOf(av.Interface())
// 要复制哪些字段
_fields := make([]string, 0)
if len(fields) > 0 {
_fields = fields
} else {
for i := 0; i < bv.NumField(); i++ {
_fields = append(_fields, bt.Field(i).Name)
}
}
if len(_fields) == 0 {
return
}
// 复制
for i := 0; i < len(_fields); i++ {
name := _fields[i]
f := av.Elem().FieldByName(name)
bValue := bv.FieldByName(name)
// a中有同名的字段并且类型一致才复制
if f.IsValid() && f.Kind() == bValue.Kind() && f.CanSet() {
f.Set(bValue)
}
}
return
}
golang 自己实现Rpc调用客户端
package components
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/astaxie/beego/config"
"io"
"io/ioutil"
"net/http"
)
var (
defaultRpcConf config.Configer
)
type RpcResponse struct {
Id string `json:"id"`
Code string `json:"code"`
Message string `json:"message"`
Type string `json:"type"`
Payload interface{} `json:"payload"`
}
//发起远程调用
//TODO 鉴权可以在这里进行
func Request(url string, data io.Reader, payload* interface{}) (*RpcResponse, error) {
req,err := http.NewRequest("POST", url, data)
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
str := string(body[:])
fmt.Print(str)
v := RpcResponse{Payload: payload}
json.Unmarshal(body, &v)
return &v,nil
}
func Rpc(name string, data interface{}, payload interface{}) (*RpcResponse, error) {
url := defaultRpcConf.String("urls::test")
requestJson, err := json.Marshal(data)
if err != nil {
return nil, err
}
p, e := Request(url, bytes.NewReader(requestJson), &payload)
if e != nil {
return nil, e
}
return p, nil
}
//rpc 简单调用方法
func RpcWithPayload(name string, data interface{}, payload interface{}) error {
p, e := Rpc(name, data, payload)
if e != nil {
return e
}
//检查rpc返回状态码
if p.Code != "0" {
return errors.New("error code: " + p.Code + " Message: " + p.Message)
}
return nil
}
func init() {
conf, err := config.NewConfig("ini", "../conf/rpc.conf")
if err != nil {
panic(err)
}
defaultRpcConf = conf
}
cronbot 安装使用
安装官方教程安装cronbot命令。
常见问题:
Q1:No module named ‘requests.packages.urllib3’
请使用 (pip install requests urllib3 –force –upgrade)升级urllib3版本既可解决。
Q2:ImportError: ‘pyOpenSSL’ module missing required functionality. Try upgrading to v0.14 or newer.
请使用 (pip uninstall pyOpenSSL; pip install pyOpenSSL==16.2.0) 安装16.2.0版本的pyOpenSSL才可以,最新的17.0还是会出现报错
批量操作服务器脚本
该脚本的前提是在 ~/.ssh/config 已经配置好的情况下使用
#!/bin/bash ##################################################################### ######## 服务器批量操作脚本 ######## Email: i@liufang.org.cn ##################################################################### for host in `cat ~/.ssh/config |grep 'Host '|awk -F ' ' '{print $2}'` do echo -e "\033[34m\r\nexec command:$1 at host $host \033[0m" ssh $host $@ done
spark简单实例
private static void runBasicDataFrameExample(SparkSession spark) throws AnalysisException { // $example off:global_temp_view$ Properties prop = new Properties(); prop.setProperty("user","root"); prop.setProperty("password","admin"); //read data form mysql novel article Dataset<Row> df2 = spark.read().jdbc("jdbc:mysql://localhost:3306/igogo", "novel_info", prop); df2.createGlobalTempView("novel"); //Dataset<Row> sqlresult = spark.sql("SELECT to_char(create_time, 'yyyymmdd hh24') as d, count(*) FROM global_temp.novel group by d"); // df2.sqlContext().udf().register("d", (String s) -> s.length(), DataType.INTEGER); spark.udf().register("d", (String s) -> { return s.length();}, DataTypes.IntegerType); Dataset<Row> sqlresult = spark.sql("SELECT id, d(title) as myfunc FROM global_temp.novel having myfunc>10"); sqlresult.show(); //config mysql // spark.newSession().sql("SELECT * FROM global_temp.people").write().mode(org.apache.spark.sql.SaveMode.Append) // .jdbc("jdbc:mysql://localhost:3306/test", "test", prop); }
maven 镜像配置
国内网络环境决定, 很多开发工作, 软件包管理软件需要配置镜像才能相对正常工作, maven也不例外。
添加配置文件 ~/.m2/settings.xml
<?xml version="1.0" encoding="UTF-8"?> <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> <mirrors> <!-- 阿里云仓库 --> <mirror> <id>alimaven</id> <mirrorOf>central</mirrorOf> <name>aliyun maven</name> <url>http://maven.aliyun.com/nexus/content/repositories/central/</url> </mirror> <!-- 中央仓库1 --> <mirror> <id>repo1</id> <mirrorOf>central</mirrorOf> <name>Human Readable Name for this Mirror.</name> <url>http://repo1.maven.org/maven2/</url> </mirror> <!-- 中央仓库2 --> <mirror> <id>repo2</id> <mirrorOf>central</mirrorOf> <name>Human Readable Name for this Mirror.</name> <url>http://repo2.maven.org/maven2/</url> </mirror> </mirrors> </settings>
在微服务架构中, 多API请求往往占据很长时间, 特别是对于那些没有相互依赖的api请求, 我们可以调整为并发请求,如此能大大节约api请求总时间。
下面的函数是利用curl并发请求改写的一个函数, 可以很好的解决多api请求的问题。
function multi_curl($requests = []) { $response = []; $hander = []; $mh = curl_multi_init(); foreach($requests as $id=>$item) { if(empty($item['url'])) { throw new \Exception('基本参数URL不能为空'); } $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $item['url']); curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_RETURNTRANSFER,1); curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); curl_setopt($ch, CURLOPT_HTTPHEADER, array('Expect: ')); if(defined('CURLOPT_IPRESOLVE') && defined('CURL_IPRESOLVE_V4')){ curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); } curl_setopt($ch, CURLOPT_TIMEOUT, isset($item['timeout']) ? $item['timeout'] : 3); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, isset($item['method']) ? $item['method'] : 'GET'); //设置请求参数 if(isset($item['data'])) { curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($item['data'])); } curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_DIGEST); //定义api请求认证信息 if(defined('API_USERNAME') && defined('API_PASSWORD')) { curl_setopt($ch, CURLOPT_USERPWD, API_USERNAME . ":" . API_PASSWORD); } curl_multi_add_handle($mh,$ch); $hander[$id] = $ch; } $running=null; do { usleep(1000); curl_multi_exec($mh,$running); } while ($running > 0); //get content and remote handle foreach($hander as $id=>$ch) { $response[$id] = curl_multi_getcontent($ch); //检查结果是否需要转换为json, 默认转换 $toArray = isset($requests[$id]['to_array']) ? $requests[$id]['to_array'] == true : true; if($toArray) { //$response[$id] = json_decode(trim($data, chr(239) . chr(187) . chr(191)), true); $response[$id] = json_decode($response[$id], true); } curl_multi_remove_handle($mh, $ch); } curl_multi_close($mh); //检查是否进行数组转换, 默认转换为数组 return $response; }
配置docker国内镜像加速
在国内如果没拉专线大部分都是很慢很慢, 无法正常使用docker, 但是我们可以通过镜像地址来减轻这种现象。
配置信息(/etc/default/docker):
# Docker Upstart and SysVinit configuration file # # THIS FILE DOES NOT APPLY TO SYSTEMD # # Please see the documentation for "systemd drop-ins": # https://docs.docker.com/engine/admin/systemd/ # # Customize location of Docker binary (especially for development testing). #DOCKERD="/usr/local/bin/dockerd" # Use DOCKER_OPTS to modify the daemon startup options. #DOCKER_OPTS="--dns 8.8.8.8 --dns 8.8.4.4" #阿里云镜像 DOCKER_OPTS="--registry-mirror=https://4xle30hc.mirror.aliyuncs.com" #官方中国镜像 #DOCKER_OPTS="--registry-mirror=https://registry.docker-cn.com" #163镜像 #DOCKER_OPTS="--registry-mirror=http://hub-mirror.c.163.com" # If you need Docker to use an HTTP proxy, it can also be specified here. #export http_proxy="http://127.0.0.1:3128/" # This is also a handy place to tweak where Docker's temporary files go. #export DOCKER_TMPDIR="/mnt/bigdrive/docker-tmp"
修改配置后请重启docker服务:
sudo service docker restart