package business import ( "encoding/json" "fmt" "github.com/360EntSecGroup-Skylar/excelize" "github.com/google/uuid" "github.com/shopspring/decimal" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" "math" "strconv" "strings" "sync" "time" "wss-pool/cmd/common" "wss-pool/config" "wss-pool/dictionary" "wss-pool/internal" "wss-pool/internal/data" red "wss-pool/internal/redis" "wss-pool/logging/applogger" "wss-pool/pkg/model" "wss-pool/pkg/model/stock" ) var TapsMapMu = map[int32]string{ 1: "NYSE", 2: "NYSEARCA", 3: "NASDAQ", } var TapsMapMuNew = map[string]string{ "XNYS": "NYSE", "BATS": "NYSE", "XASE": "NYSE", "ARCX": "NYSE", "XNAS": "NASDAQ", } var CountryEx = map[string]string{ "Thailand": "SET", "Indonesia": "IDX", "Malaysia": "MYX", "Singapore": "SGX", "HongKong": "HKEX", "Japan": "TSE", } var StockClosingPrice = map[string]string{ "US": "Stock:US:ClosePrice", //美股当天闭盘价 "USNew": "Stock:US:CloseNewPrice", //美股实时落盘价 "USBeforeClose": "Stock:US:BeforeClose", //美股上一次闭盘价 "Malaysia": "Stock:Malaysia:ClosePrice", //马股当天闭盘价 "MalaysiaNew": "Stock:Malaysia:CloseNewPrice", //马股实时落盘价 "MalaysiaBeforeClose": "Stock:Malaysia:BeforeClose", //马股上一次闭盘价 "Thailand": "Stock:Thailand:ClosePrice", //泰股当天闭盘价 "ThailandNew": "Stock:Thailand:CloseNewPrice", //泰股实时落盘价 "ThailandBeforeClose": "Stock:Thailand:BeforeClose", //泰股上一次闭盘价 "Indonesia": "Stock:Indonesia:ClosePrice", //印尼股当天闭盘价 "IndonesiaNew": "Stock:Indonesia:CloseNewPrice", //印尼股实时落盘价 "IndonesiaBeforeClose": "Stock:Indonesia:BeforeClose", //印尼股上一次闭盘价 "India": "Stock:India:ClosePrice", //印度股当天闭盘价 "IndiaNew": "Stock:India:CloseNewPrice", //印度股实时落盘价 "IndiaBeforeClose": "Stock:India:BeforeClose", //印度股上一次闭盘价 "StockIndex": "Stock:Index:ClosePrice", //指数当天闭盘价 "StockIndexNew": "Stock:Index:CloseNewPrice", //指数实时落盘价 "StockIndexBeforeClose": "Stock:Index:BeforeClose", //指数股上一次闭盘价 "Singapore": "Stock:Singapore:ClosePrice", //新加坡当天闭盘价 "SingaporeNew": "Stock:Singapore:CloseNewPrice", //新加坡股实时落盘价 "SingaporeBeforeClose": "Stock:Singapore:BeforeClose", //新加坡股上一次闭盘价 "HongKong": "Stock:HongKong:ClosePrice", //港股当天闭盘价 "HongKongNew": "Stock:HongKong:CloseNewPrice", //港股实时落盘价 "HongKongBeforeClose": "Stock:HongKong:BeforeClose", //港股上一次闭盘价 "OptionUS": "Stock:OptionUS:ClosePrice", //期权美股当天闭盘价 "OptionUSNew": "Stock:OptionUS:CloseNewPrice", //期权美股实时落盘价 "OptionUSBeforeClose": "Stock:OptionUS:BeforeClose", //期权美股上一次闭盘价 "OptionIndia": "Option:India:ClosePrice", //期权印度股当天闭盘价 "OptionIndiaNew": "Option:India:CloseNewPrice", //期权印度股实时落盘价 "OptionIndiaBeforeClose": "Option:India:BeforeClose", //期权印度股上一次闭盘价 "OptionIndiaPrice": "Option:India:List", "UK": "Stock:UK:ClosePrice", //英股当天闭盘价 "UKNew": "Stock:UK:CloseNewPrice", //英股实时落盘价 "UKBeforeClose": "Stock:UK:BeforeClose", //英股上一次闭盘价 "Germany": "Stock:Germany:ClosePrice", //德股当天闭盘价 "GermanyNew": "Stock:Germany:CloseNewPrice", //德股实时落盘价 "GermanyBeforeClose": "Stock:Germany:BeforeClose", //德股上一次闭盘价 "France": "Stock:France:ClosePrice", //法股当天闭盘价 "FranceNew": "Stock:France:CloseNewPrice", //法股实时落盘价 "FranceBeforeClose": "Stock:France:BeforeClose", //法股上一次闭盘价 "Brazil": "Stock:Brazil:ClosePrice", //巴西当天闭盘价 "BrazilNew": "Stock:Brazil:CloseNewPrice", //巴西实时落盘价 "BrazilBeforeClose": "Stock:Brazil:BeforeClose", //巴西上一次闭盘价 "Japan": "Stock:Japan:ClosePrice", //日本当天闭盘价 "JapanNew": "Stock:Japan:CloseNewPrice", //日本实时落盘价 "JapanBeforeClose": "Stock:Japan:BeforeClose", //日本上一次闭盘价 } var ( mutexMap = sync.RWMutex{} NewPriceMap = make(map[string]string) IsRealFlase int = 2 StockCountryMap = map[string]string{ "US": "TSLA.US", "Thailand": "SET:AAV.Thailand", "Indonesia": "IDX:GOTO.Indonesia", "India": "NSE:SUZLON.India", "Malaysia": "MYX:MAYBANK.Malaysia", "Singapore": "SGX:D05.Singapore", "HongKong": "HKEX:9888.HongKong", "UK": "LSE:BRBY.UK", "France": "EURONEXT:RMS.France", "Germany": "FWB:TMV.Germany", "Brazil": "BMFBOVESPA:BBAS3.Brazil", "Japan": "TSE:1380.Japan", } StockCountryIsValidMap = map[string]bool{ "US": false, "Thailand": false, "Indonesia": false, "India": false, "Malaysia": false, "Singapore": false, "HongKong": false, "UK": false, "France": false, "Germany": false, "Brazil": false, "Japan": false, } StockIsValidMutex = sync.RWMutex{} ) // UpdateStockUS Daily update of US stock list data【ws.eodhistoricaldata.com】 func UpdateStockUS() { red.RedisInitMap(common.GetRedisDBMore(config.Config.Redis.DbMore)) filter := bson.M{"Country": "US", "IsReal": bson.M{"$ne": IsRealFlase}} dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } day := common.TimeToNows().Weekday() if day == time.Sunday { applogger.Debug("周日 无数据") return } for _, value := range dateList.([]primitive.M) { code := TypeCheck(value["Code"]) //定时任务 最长休假 算 10 天 // today := common.TimeToNows().AddDate(0, 0, -10).Unix() // eodModel, _ := PreviousClose(code) // if len(eodModel.Results) <= 0 { // applogger.Error("close price null") // continue // } // dateStrs := common.ConvertToTimezones(eodModel.Results[0].T) // if dateStrs.Unix() < today { // applogger.Error(code, dateStrs, "超出时间范围") // //updateYesterdayClose(code, "US") // continue // } // yesterday := dateStrs //Loop: // yesterday = yesterday.AddDate(0, 0, -1) // yesterdayClose, _ := UsData(code, yesterday.Format("2006-01-02")) // if yesterdayClose == "" { // goto Loop // } Loop: res := GetFinnhubBeforClose(code) if strings.Contains(res.Error, "API limit reached") { //防止被限制 time.Sleep(6 * time.Second) goto Loop } if !res.C.GreaterThan(decimal.Zero) { applogger.Error(code, "close price null") continue } filterS := bson.D{{"Code", bson.M{ "$eq": code, }}, {"Country", bson.M{ "$eq": "US", }}} dp, _ := res.DP.Float64() //股价百分比变化 updateData := bson.M{ "$set": bson.M{ "BeforeClose": res.PC.String(), "DateStr": common.ConvertToTimezone(res.T * 1000), "DP": dp, "YesterdayClose": res.C.String()}} red.HsetMap(StockClosingPrice["US"], code, res.C.String()) red.HsetMap(StockClosingPrice["USBeforeClose"], code, res.PC.String()) red.HsetMap(StockClosingPrice["USNew"], code, "0") applogger.Debug("update data info:%v", updateData) if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } // TODO: 更改副表 没有公用数据的服务不需要 暂用redis 参数 dbs := common.GetRedisDBMore(config.Config.Redis.DbMore) if len(dbs) > 1 { applogger.Info("StockListAdd update table db", dbs[1]) data.MgoUpdateOne(fmt.Sprintf("%s%s", data.StockList, dbs[1]), filterS, updateData) } } } func UpdateOpenPrice() { red.RedisClient = red.RedisInit(config.Config.Redis.DbTen) stocks, _, pageTotal := GetStockAll("US", 1, UsPageSize) wg := sync.WaitGroup{} end := common.TimeToNows().Format("2006-01-02") for _, value := range stocks { wg.Add(1) go func(end, code string) { defer wg.Done() res := GetBeforClose(code, "1", "day", end, end) if len(res.Results) > 0 { applogger.Debug("US open price %v ,%v", code, res.Results[0].O.String()) red.Hset(StockClosingPrice["US"], code, res.Results[0].O.String()) } }(end, value.Code) } wg.Wait() for i := int64(2); i <= pageTotal; i++ { stocks, _, _ := GetStockAll("US", i, UsPageSize) wg := sync.WaitGroup{} for _, value := range stocks { wg.Add(1) go func(end, code string) { defer wg.Done() res := GetBeforClose(code, "1", "day", end, end) if len(res.Results) > 0 { applogger.Debug("US open price %v ,%v", code, res.Results[0].O.String()) red.Hset(StockClosingPrice["US"], code, res.Results[0].O.String()) } }(end, value.Code) } wg.Wait() } } func updateYesterdayClose(code, country string) { filterS := bson.D{{"Code", bson.M{ "$eq": code, }}, {"Country", bson.M{ "$eq": country, }}} updateData := bson.M{ "$set": bson.M{ "YesterdayClose": ""}} red.Hset(StockClosingPrice[country], code, "0") red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", country)], code, "0") red.Hset(StockClosingPrice[fmt.Sprintf("%sBeforeClose", country)], code, "0") if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } } func UpdateYesterdayCloseIs(code, country, yesterdayClose, beforeClose string) { filterS := bson.D{{"Code", bson.M{ "$eq": code, }}, {"Country", bson.M{ "$eq": country, }}} updateData := bson.M{ "$set": bson.M{ "YesterdayClose": yesterdayClose}} red.Hset(StockClosingPrice[country], code, yesterdayClose) red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", country)], code, 0) red.Hset(StockClosingPrice[fmt.Sprintf("%sBeforeClose", country)], code, beforeClose) if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } } func GetBeforClose(code string, multiplier, timespan, start, end string) stock.PreviousCloseResponse { var eodModel stock.PreviousCloseResponse url := fmt.Sprintf("https://%v/v2/aggs/ticker/%v/range/%v/%v/%v/%v?adjusted=true&sort=desc&%v", config.Config.ShareGather.PolygonHost, code, multiplier, timespan, start, end, fmt.Sprintf("apiKey=%v", config.Config.ShareGather.PolygonKey)) applogger.Debug("UrlHttp getBeforClose info: %v", url) bodyStr, err := internal.HttpGet(url) if err != nil { applogger.Error("Failed to query data:%v", err) return eodModel } if err = json.Unmarshal([]byte(bodyStr), &eodModel); err != nil { applogger.Error("eodModel json Unmarshal err: %v", err) return eodModel } return eodModel } func GetFinnhubBeforClose(code string) stock.PreviousCloseRes { var eodModel stock.PreviousCloseRes url := fmt.Sprintf("https://%vquote?symbol=%s&token=%s", config.Config.FinnhubUs.FinnhubHost, code, config.Config.FinnhubUs.FinnhubKey) applogger.Debug("UrlHttp getBeforClose info: %v", url) bodyStr, err := internal.HttpGet(url) fmt.Println(bodyStr) if err != nil { applogger.Error("Failed to query data:%v", err) return eodModel } if err = json.Unmarshal([]byte(bodyStr), &eodModel); err != nil { applogger.Error("eodModel json Unmarshal err: %v", err) return eodModel } return eodModel } // 更新交易所 func UpdateStockUSTape() { filter := bson.M{"Country": "US"} dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } for _, value := range dateList.([]primitive.M) { code := TypeCheck(value["Code"]) eodModel, _ := TradesTape(code) filterS := bson.D{{"Code", bson.M{ "$eq": code, }}, {"Country", bson.M{ "$eq": "US", }}} if len(eodModel.Results) <= 0 { continue } updateData := bson.M{ "$set": bson.M{ "Code": code, "Tape": eodModel.Results[0].Tape}} applogger.Debug("update data info:%v", updateData) if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } } } // 推送 func StockPyWs(param model.StockParam, country string) { param.Token = "" msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return } //applogger.Info("last date info: %v", string(msgStr)) // Write to Redis for broadcasting red.PublishMap(fmt.Sprintf("%s.%s", param.Symbol, country), string(msgStr)) } func IsPriceTime(symbol, price, country string) bool { //fmt.Println(symbol, price, country) mutexMap.RLock() val := NewPriceMap[fmt.Sprintf("%s-%s", country, symbol)] mutexMap.RUnlock() if val != "" { strs := strings.Split(val, "-") if len(strs) > 0 { timeInt, _ := strconv.ParseInt(strs[1], 10, 64) if strs[0] == price && common.TimeToNow() < timeInt { return false } } } mutexMap.Lock() NewPriceMap[fmt.Sprintf("%s-%s", country, symbol)] = fmt.Sprintf("%s-%d", price, common.TimeToNow()+30) mutexMap.Unlock() return true } // 推送 func StockPyWsStockIndex(param model.StockIndexParam, country string) { param.Token = "" param.IsStockIndex = true msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return } //applogger.Info("last date info: %v", string(msgStr)) // Write to Redis for broadcasting red.PublishMap(fmt.Sprintf("%s.%s", param.StockCode, country), string(msgStr)) } // 期权列表推送 func StockPyWsOptionList(param model.OptionPolygon, country string) string { param.IsOptionList = true msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return "" } //applogger.Info("last date info: %v", string(msgStr)) // Write to Redis for broadcasting red.PublishMap(fmt.Sprintf("%s.%s", param.Stock, country), string(msgStr)) return string(msgStr) } // 期权详情推送 func StockPyWsOptionInfo(param model.OptionInfoParam, country string) string { param.IsOptionInfo = true param.Token = "" msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return "" } //applogger.Info("StockPyWsOptionInfo info: %v", string(msgStr)) // Write to Redis for broadcasting red.PublishMap(fmt.Sprintf("%s.%s", param.Stock, country), string(msgStr)) return string(msgStr) } // 期权详情Exchange推送 func StockPyWsOptionInfoExchange(param model.OptionInfoExchange, country string) string { param.IsOptionInfo = true msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return "" } applogger.Info("StockPyWsOptionInfoExchange info: %v", string(msgStr)) red.PublishMap(fmt.Sprintf("%s.%s", param.Stock, country), string(msgStr)) return string(msgStr) } func OptionResPrice(param model.StrikePrice, key, code string, isClose bool, optionDate, expiration string) string { result := model.StrikePrice{} resStr, _ := red.Hget(code, key) param.DueDate = common.OptionTime(expiration) if resStr != "" { json.Unmarshal([]byte(resStr), &result) param.BeforeClose = result.BeforeClose param.YesterdayClose = result.YesterdayClose param.CloseDate = result.CloseDate if isClose && optionDate != result.CloseDate { //多次传闭盘价 param.BeforeClose = result.YesterdayClose param.YesterdayClose = param.Price param.Price = "0" param.CloseDate = optionDate } else if isClose && optionDate == result.CloseDate { param.BeforeClose = result.BeforeClose param.YesterdayClose = param.Price param.Price = "0" } } msgStr, err := json.Marshal(param) if err != nil { applogger.Error("json.Marshal err: %v", err) return "" } red.HsetMap(code, key, string(msgStr)) return string(msgStr) } func DelOptionHash() { red.RedisClient = red.RedisInit(config.Config.Redis.DbTen) keys := red.Scan("Option:India:List") for _, key := range keys { res, _ := red.HGetAll(key) for k, val := range res { result := model.StrikePrice{} json.Unmarshal([]byte(val), &result) if common.TimeToNow() >= (result.DueDate + 24*60*60) { applogger.Info("DelOptionHash : expireTime %v", result.DueDate, key, k) red.HDel(key, k) } } } } func UpdateStockBeforeClose(symbol, price, country string) { //if val, ok := NewPriceMap[fmt.Sprintf("%s-%s", country, symbol)]; ok && val == price { // applogger.Info("new price", fmt.Sprintf("%s-%s", country, symbol), price) // return //} red.HsetMap(StockClosingPrice[fmt.Sprintf("%sNew", country)], symbol, price) //NewPriceMap[fmt.Sprintf("%s-%s", country, symbol)] = price } // UpdateStockUSBak api.polygon.io Update US stock list data func UpdateStockUSBak() { yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") before := time.Now().AddDate(0, 0, -2).Format("2006-01-02") // 从mongodb中获取股票代码 filter := bson.M{"Country": "USA", "YesterdayClose": bson.M{"$ne": ""}, "BeforeClose": bson.M{"$ne": ""}} dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } for _, value := range dateList.([]primitive.M) { code := TypeCheck(value["Code"]) yesterdayClose, err := UsData(code, yesterday) if err != nil || len(yesterdayClose) == 0 { applogger.Error("yesterdayClose ConstructorData info err: %v", err) yesterdayClose = TypeCheck(value["YesterdayClose"]) } beforeClose, err := UsData(code, before) if err != nil || len(beforeClose) == 0 { applogger.Error("beforeClose ConstructorData info err: %v", err) beforeClose = TypeCheck(value["BeforeClose"]) } applogger.Debug("data info:%v-----%v-----%v", code, yesterdayClose, beforeClose) filterS := bson.D{{"Code", bson.M{ "$eq": code, }}} updateData := bson.M{ "$set": bson.M{ "Code": code, "YesterdayClose": yesterdayClose, "BeforeClose": beforeClose}} applogger.Debug("update data info:%v", updateData) if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } } } // UsData Obtaining the closing price of US stocks through time【api.polygon.io】 func UsData(code, date string) (string, error) { url := fmt.Sprintf("https://%v/v1/open-close/%v/%v?apiKey=%v&adjusted=true", config.Config.ShareGather.PolygonHost, code, date, config.Config.ShareGather.PolygonKey) bodyStr, err := internal.HttpGet(url) if err != nil { applogger.Error("Failed to query data:%v", err) return "", err } applogger.Debug("url info:%v", url) if strings.Contains(bodyStr, ",\"message\":") { return "", err } var eodModel stock.UsDateClose if err = json.Unmarshal([]byte(bodyStr), &eodModel); err != nil { applogger.Error("Unmarshal err: %v---%v", eodModel.Symbol, err) return "", err } return eodModel.Close.String(), err } func UsNewPrice(code string) { red.RedisClient = red.RedisInit(config.Config.Redis.DbEleven) //定时任务 最长休假 算 10 天 today := common.TimeToNows().AddDate(0, 0, -10).Unix() eodModel, _ := PreviousClose(code) if len(eodModel.Results) <= 0 { applogger.Error("close price null") return } fmt.Println(eodModel) dateStrs := common.ConvertToTimezones(eodModel.Results[0].T) if dateStrs.Unix() < today { applogger.Error(code, dateStrs, "超出时间范围") //updateYesterdayClose(code, "US") return } yesterday := dateStrs Loop: yesterday = yesterday.AddDate(0, 0, -1) yesterdayClose, _ := UsData(code, yesterday.Format("2006-01-02")) if yesterdayClose == "" { goto Loop } filterS := bson.D{{"Code", bson.M{ "$eq": code, }}, {"Country", bson.M{ "$eq": "US", }}} updateData := bson.M{ "$set": bson.M{ "Code": code, "BeforeClose": yesterdayClose, "DateStr": dateStrs.Format("2006-01-02"), "Vol": eodModel.Results[0].V.IntPart(), "YesterdayClose": eodModel.Results[0].C.String()}} red.Hset(StockClosingPrice["US"], code, eodModel.Results[0].C.String()) red.Hset(StockClosingPrice["USBeforeClose"], code, yesterdayClose) red.Hset(StockClosingPrice["USNew"], code, "0") applogger.Debug("update data info:%v", updateData) if err := data.MgoUpdateOne(data.StockList, filterS, updateData); err != nil { applogger.Error("MgoBulkWrite update err:%v", err) return } } // 倒数据 func SymbolToStock(country string) { if country == "" { return } red.RedisClient = red.RedisInit(config.Config.Redis.DbTen) config.Config.Mongodb.DbHost = "10.154.0.5" data.Mgo_init(config.Config.Mongodb) filter := bson.M{"Country": country} if country == "StockIndex" { filter = bson.M{} } res := make([]stock.StockPolygonS, 0) tableName := data.StockList if country == "StockIndex" { tableName = data.StockIndexList } data.MgoFindRes(tableName, filter, &res) config.Config.Mongodb.DbHost = "10.154.0.10" data.Mgo_init(config.Config.Mongodb) var dataList []mongo.WriteModel fmt.Println(len(res)) for _, value := range res { filter := bson.D{{"Code", bson.M{ "$eq": value.Code, }}, {"Country", bson.M{ "$eq": value.Locale, }}} update := bson.D{{"$set", bson.D{ {"Code", value.Code}, {"Name", value.Name}, {"Country", value.Locale}, {"Exchange", value.PrimaryExchange}, {"Currency", value.Currency}, {"Intro", value.Intro}, {"Type", value.Type}, {"Cik", value.CIK}, {"ShareClassFigi", value.ShareClassFigi}, {"YesterdayClose", value.YesterdayClose}, {"BeforeClose", value.BeforeClose}, {"Tape", value.Tape}, {"State", value.State}, {"NumericCode", value.NumericCode}, {"DateStr", value.DateStr}, {"Sort", value.Sort}, {"LogoUrl", value.LogoUrl}}}} models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) dataList = append(dataList, models) //red.Hset(StockClosingPrice[country], value.Code, value.YesterdayClose) //red.Hset(StockClosingPrice[fmt.Sprintf("%sBeforeClose", country)], value.Code, value.BeforeClose) //red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", country)], value.Code, "0") } // fmt.Println(2222) if err := data.MgoBulkWrite(tableName, dataList); err != nil { applogger.Error("MgoInsertMany err:%v", err) return } } // SymbolToStockList // // @Description: 将旧的美股数据导入新的项目 // @param country func SymbolToStockList(country string) { config.Config.Mongodb.DbHost = "35.189.116.242" data.Mgo_init(config.Config.Mongodb) res := make([]stock.StockListBak, 0) data.MgoFindRes(data.StockList, bson.M{"Country": country}, &res) applogger.Debug("查询股票列表:%v", len(res)) config.Config.Mongodb.DbHost = "104.198.117.66" data.Mgo_init(config.Config.Mongodb) var dataList []mongo.WriteModel for _, value := range res { filter := bson.D{{"Code", bson.M{ "$eq": value.Code, }}, {"Country", bson.M{ "$eq": value.Country, }}} update := bson.D{{"$set", bson.D{ {"Country", value.Country}, {"Code", value.Code}, {"BeforeClose", value.BeforeClose}, {"Cik", value.Cik}, {"CompositeFigi", value.CompositeFigi}, {"Currency", value.Currency}, {"Exchange", value.Exchange}, {"Name", value.Name}, {"ShareClassFigi", value.ShareClassFigi}, {"Type", value.Type}, {"YesterdayClose", value.YesterdayClose}, {"DP", value.DP}, {"DateStr", value.DateStr}}}} models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) dataList = append(dataList, models) } applogger.Debug("倒入数据:%v", len(dataList)) if err := data.MgoBulkWrite(data.StockList, dataList); err != nil { applogger.Error("MgoInsertMany err:%v", err) return } } // 倒数据 func SymbolToStockInfo(country string) { if country == "" { return } config.Config.Mongodb.DbHost = "34.93.29.102" data.Mgo_init(config.Config.Mongodb) config.Config.Mongodb.DbHost = "47.236.120.73" client := data.Mgo_inits(config.Config.Mongodb) fmt.Println(country) for _, period := range dictionary.StockSouthAsiaListTimes { tableName := data.GetStockSouthAsiaTableName(country, period) if country == "StockIndex" { tableName = data.GetStockIndixKlineTableName(period) } fmt.Println(tableName) res, _, pageTotal := GetStockIndoAll(tableName, 1, 100, period) var dataList []mongo.WriteModel for _, v := range res { filter := bson.M{"timestamp": bson.M{"$eq": v.Ts}, "symbol": bson.M{"$eq": v.Symbol}} update := bson.D{{"$set", bson.D{ {"symbol", v.Symbol}, {"stock_code", v.StockCode}, {"stock_name", v.StockName}, {"open_price", v.OpenPrice}, {"high_price", v.HighPrice}, {"low_price", v.LowPrice}, {"close_price", v.ClosePrice}, {"vol", v.Vol}, {"turnover_price_total", v.TurnoverPriceTotal}, {"price_total", v.PriceTotal}, {"price_code", v.PriceCode}, {"country", v.Country}, {"timestamp", v.Ts}, }}} models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) dataList = append(dataList, models) } if err := data.MgoBulkWrites(client, tableName, dataList); err != nil { applogger.Error("stock MgoInsertMany err:%v", err) } fmt.Println(pageTotal) //for i := int64(2); i <= pageTotal; i++ { // res, _, _ := GetStockIndoAll(tableName, i, 100, period) // fmt.Println(period, i) // for _, v := range res { // filter := bson.M{"timestamp": bson.M{"$eq": v.Ts}, "symbol": bson.M{"$eq": v.Symbol}} // update := bson.D{{"$set", bson.D{ // {"symbol", v.Symbol}, // {"stock_code", v.StockCode}, // {"stock_name", v.StockName}, // {"open_price", v.OpenPrice}, // {"high_price", v.HighPrice}, // {"low_price", v.LowPrice}, // {"close_price", v.ClosePrice}, // {"vol", v.Vol}, // {"turnover_price_total", v.TurnoverPriceTotal}, // {"price_total", v.PriceTotal}, // {"price_code", v.PriceCode}, // {"country", v.Country}, // {"timestamp", v.Ts}, // }}} // models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) // dataList = append(dataList, models) // } // if err := data.MgoBulkWrites(client, tableName, dataList); err != nil { // applogger.Error("stock MgoInsertMany err:%v", err) // } //} } } func GetStockIndoAll(tableName string, pageNum, pageSize int64, period string) ([]model.StockMogoParams, int64, int64) { filter := bson.M{} if period == "1hour" { filter = bson.M{"timestamp": bson.M{"$gte": time.Now().AddDate(0, 0, -20).UnixMilli()}} } else if period == "1day" { filter = bson.M{"timestamp": bson.M{"$gte": time.Now().AddDate(0, -6, 0).UnixMilli()}} } projection := bson.M{} res := make([]model.StockMogoParams, 0) total, _ := data.MgoFindTotal(tableName, filter) data.MgoPagingFindStructProjection(tableName, filter, projection, pageSize, pageNum, -1, &res) return res, total, int64(math.Ceil(float64(total) / float64(pageSize))) } func SymbolNews(country string) { config.Config.Mongodb.DbHost = "104.198.117.66" data.Mgo_init(config.Config.Mongodb) filter := bson.M{"country": country} res := make([]model.StockNews, 0) data.MgoFindRes(data.StockNews, filter, &res) config.Config.Mongodb.DbHost = "35.186.148.111" data.Mgo_init(config.Config.Mongodb) var bsonEod []interface{} for _, v := range res { bsonEod = append(bsonEod, bson.D{ {"country", v.Country}, {"pubdate", v.Pubdate}, {"title", v.Title}, {"link", v.Link}, {"source", v.Source}, {"code", v.Code}, }) } if err := data.MgoInsertMany(data.StockNews, bsonEod); err != nil { applogger.Error("MgoInsertMany info err: %v", err) return } //} fmt.Println(1111) } // 更改股票code结构(美股不需要更改) func UpdateStockCode() { red.RedisClient = red.RedisInit(config.Config.Redis.DbEleven) filter := bson.M{} dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } for _, value := range dateList.([]primitive.M) { if value["Code"] == nil || value["Country"] == nil || (value["Tape"] == nil && value["Exchange"] == nil) { continue } code := value["Code"].(string) var exchange, symbol string country := value["Country"].(string) if country == "US" { exchange = TapsMapMu[value["Tape"].(int32)] symbol = code } else { exchange = value["Exchange"].(string) if exchange == "" && country != "India" { exchange = CountryEx[country] } if exchange == "" { applogger.Error("没有值", country, symbol, exchange) continue } symbol = fmt.Sprintf("%s:%s", exchange, code) red.HDel(StockClosingPrice[country], code) red.Hset(StockClosingPrice[country], symbol, value["YesterdayClose"]) red.HDel(StockClosingPrice[fmt.Sprintf("%sBeforeClose", country)], code) red.Hset(StockClosingPrice[fmt.Sprintf("%sBeforeClose", country)], symbol, value["BeforeClose"]) red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", country)], code, "0") red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", country)], symbol, "0") } filterList := bson.M{"Code": code, "Country": country} if err = data.MgoUpdateOne( data.StockList, filterList, bson.D{{"$set", bson.D{ {"Code", symbol}, {"Exchange", exchange}, {"Symbol", code}}}}); err != nil { applogger.Error(symbol, err) } applogger.Info(symbol, exchange, code, country) } } func UpdateStockUsCode() { red.RedisClient = red.RedisInit(config.Config.Redis.DbEleven) filter := bson.M{"Country": "US"} dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } for _, value := range dateList.([]primitive.M) { if value["Code"] == nil || value["Country"] == nil || value["Exchange"] == nil { continue } codeList := strings.Split(value["Code"].(string), ".") if len(codeList) == 0 { continue } code := codeList[0] var exchange, symbol string country := value["Country"].(string) if country == "US" { exchange = TapsMapMuNew[value["Exchange"].(string)] symbol = code } if err = data.MgoUpdateOne( data.StockList, bson.M{"Code": value["Code"].(string), "Country": country}, bson.D{{"$set", bson.D{ {"Code", symbol}, //{"Exchange", exchange}, {"Symbol", code}}}}); err != nil { applogger.Error(symbol, err) } applogger.Info(symbol, exchange, code, country) } } func UpdateStockExchange() { filter := bson.M{ "Country": "India", } dateList, err := data.MgoFind(data.StockList, filter) if err != nil { applogger.Error("MgoFind info err: %v", err) return } applogger.Info("total ", len(dateList.([]primitive.M))) var i int for _, value := range dateList.([]primitive.M) { if value["Exchange"] == nil || value["Exchange"].(string) == "" { i++ fmt.Println(i) code := value["Code"].(string) filter = bson.M{"stock_code": code} res, _ := data.MgoFinds(data.GetStockSouthAsiaTableName("India", "1day"), filter, int64(1)) if len(res) > 0 { exchange := strings.Split(res[0]["symbol"].(string), ":")[0] applogger.Info(exchange, code) filterList := bson.M{"Code": code, "Country": "India", } //applogger.Info(exchange,code) if err := data.MgoUpdateOne(data.StockList, filterList, bson.D{{"$set", bson.D{ {"Exchange", exchange}}}}); err != nil { } } else { applogger.Error(code, "no data") } } } } // 外汇代码数据信息更新 func ForexUpdateCode() { dateList, err := data.MgoFind(data.ForexListBak, bson.M{}) if err != nil { applogger.Error("MgoFind info err: %v", err) return } var codeList []string for _, value := range dateList.([]primitive.M) { code := value["code"].(string) codeList = append(codeList, code) } UrlBatchKline := "https://quote.tradeswitcher.com/quote-b-api/batch-kline?token=bf8f33c446c4494286eccaa57a2e6fac-c-app" var dataPost model.ConstructParametersPost for _, value := range codeList { dataPost.Trace = uuid.New().String() dataPost.Data.DataList = append(dataPost.Data.DataList, model.DataParameters{ Code: value, KlineType: 1, KlineTimestampEnd: 0, QueryKlineNum: 1, AdjustType: 0, }) } queryStr, err := json.Marshal(&dataPost) if err != nil { applogger.Error("解析json错误:%v", err) return } bodyStr, err := internal.HttpPost(UrlBatchKline, string(queryStr)) if err != nil { applogger.Error("读取响应失败:%v", err) return } applogger.Debug("响应内容:%v", bodyStr) var klineNew model.KlinePostReturnStruct if err = json.Unmarshal([]byte(bodyStr), &klineNew); err != nil { applogger.Error("解析失败:%v", err) return } applogger.Debug("响应内容:%v", klineNew) var dataList []mongo.WriteModel for _, v := range klineNew.Data.KlineList { filter := bson.M{ "code": v.Code, } updateData := bson.M{} for _, value := range v.KlineData { updateData["openPrice"] = value.OpenPrice updateData["highPrice"] = value.HighPrice updateData["lowPrice"] = value.LowPrice updateData["closePrice"] = value.ClosePrice updateData["timestamp"] = value.Timestamp break } update := bson.M{"$set": updateData} models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) dataList = append(dataList, models) } if len(dataList) > 0 { if err = data.MgoBulkWrite(data.ForexListBak, dataList); err != nil { applogger.Error("MgoInsertMany err:%v", err) return } } } func MalaysiaStockUpdate() { data.Mgo_init(config.Config.Mongodb) f, err := excelize.OpenFile("final.xlsx") if err != nil { fmt.Println(err) return } // 获取 Sheet1 上所有单元格 rows := f.GetRows("Sheet1") for k, row := range rows { if k == 0 { continue } fmt.Println(k, row) symbol := strings.TrimSpace(row[0]) numericCode := strings.TrimSpace(row[1]) filter := bson.M{"Code": common.GetNewCode("MYX", symbol, "Malaysia"), "Country": "Malaysia", } updateData := bson.D{{"$set", bson.D{ {"NumericCode", numericCode}}}} if err := data.MgoUpdateOne(data.StockList, filter, updateData); err != nil { applogger.Error(err.Error()) } } } func SendIndiaInfo() { for { if !common.IsOpening("India") { applogger.Debug(time.Now().Format("2006-01-02 15:04:05"), " it's not opening time -----------------------------end") continue } url := fmt.Sprintf("%s/india/spots/new/add", config.Config.SendIn.URL) param := fmt.Sprintf(`[{"stock_code":"%s","symbol":"%s","country":"india","price":%v,"vol":%d,"ts":%d,"token":"asdfsnl123jlknl3nksdf32345ln98sdfsfs8891232nsdfsdfsdfsdxcfvbhnfgh"}]`, common.GetOldCode(config.Config.SendIn.Symbol), config.Config.SendIn.Symbol, config.Config.SendIn.Price, config.Config.SendIn.Vol, time.Now().UnixMilli()) applogger.Info(param) bodyStr, err := internal.HttpPost(url, param) if err != nil { applogger.Error("Failed to query data:%v", err) } applogger.Info(bodyStr) time.Sleep(2 * time.Minute) } } func CheckStock() { time.Sleep(1 * time.Minute) red.RedisClient = red.RedisInit(config.Config.Redis.DbTen) for k, _ := range StockClosedDataList { go func(k string) { if strings.Contains(config.Config.TgBot.NoWarn, k) { fmt.Println(k, "no warn") return } else if k == "US" && !common.IsOpeningUS() { applogger.Debug(common.TimeToNows().Format("2006-01-02 15:04:05"), k, " it's not opening time -----------------------------end") return } else if (k == "Thailand" || k == "Indonesia" || k == "India" || k == "Malaysia" || k == "Singapore" || k == "HongKong" || k == "UK" || k == "France" || k == "Germany" || k == "Japan") && !common.IsOpening(k) { applogger.Debug(common.TimeToNows().Format("2006-01-02 15:04:05"), k, " it's not opening time -----------------------------end") return } fmt.Println(k) fmt.Println(StockCountryMap[k]) pubSub := red.RedisClient.Subscribe(StockCountryMap[k]) defer func() { pubSub.Close() }() _, err := pubSub.Receive() if err != nil { applogger.Error("failed to receive from control PubSub,%v", zap.Error(err)) return } go func(k string) { time.Sleep(10 * time.Minute) if !StockCountryIsValidMap[k] { common.TgBotSendMsg(fmt.Sprintf("%s %s %s 市场 %s 股票 行情异常,请注意!", common.TimeToNows().Format("2006-01-02 15:04:05"), config.Config.TgBot.Server, k, StockCountryMap[k])) } }(k) ch := pubSub.Channel() for msg := range ch { applogger.Info("Subscribe date:%v", msg.Payload) StockIsValidMutex.Lock() StockCountryIsValidMap[k] = true StockIsValidMutex.Unlock() return } }(k) } time.Sleep(13 * time.Minute) } func SymbolCode(country string) { red.RedisClient = red.RedisInit(config.Config.Redis.DbTen) data.Mgo_init(config.Config.Mongodb) filter := bson.M{"Country": country} res := make([]stock.StockPolygonS, 0) tableName := data.StockList data.MgoFindRes(tableName, filter, &res) var dataList []mongo.WriteModel //fmt.Println(len(res)) for _, value := range res { filter := bson.D{{"Code", bson.M{ "$eq": value.Code, }}, {"Country", bson.M{ "$eq": value.Locale, }}} exchange := value.PrimaryExchange if value.Locale == "Thailand" { exchange = "SET" } else if value.Locale == "Indonesia" { exchange = "IDX" } else if value.Locale == "US" { switch value.Tape { case 1: exchange = "NYSE" case 2: exchange = "NYSE-ARCA/NYSE-American" case 3: exchange = "NASDAQ" } } code := fmt.Sprintf("%s.%s", value.Code, exchange) //if exchange == "" { // fmt.Println(value.Code) //} //continue update := bson.D{{"$set", bson.D{ {"Code", code}, {"Exchange", exchange}, {"Symbol", value.Code}}}} models := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) dataList = append(dataList, models) red.Hset(StockClosingPrice[value.Locale], value.Code, value.YesterdayClose) red.Hset(StockClosingPrice[fmt.Sprintf("%sBeforeClose", value.Locale)], value.Code, value.BeforeClose) red.Hset(StockClosingPrice[fmt.Sprintf("%sNew", value.Locale)], value.Code, "0") } if err := data.MgoBulkWrite(tableName, dataList); err != nil { applogger.Error("MgoInsertMany err:%v", err) return } }