惯性聚合 高效追踪和阅读你感兴趣的博客、新闻、科技资讯
阅读原文 在惯性聚合中打开

推荐订阅源

有赞技术团队
有赞技术团队
cs.CV updates on arXiv.org
cs.CV updates on arXiv.org
P
Palo Alto Networks Blog
C
Cisco Blogs
The Hacker News
The Hacker News
T
Threatpost
S
Schneier on Security
K
Kaspersky official blog
Spread Privacy
Spread Privacy
博客园_首页
钛媒体:引领未来商业与生活新知
钛媒体:引领未来商业与生活新知
NISL@THU
NISL@THU
量子位
cs.CL updates on arXiv.org
cs.CL updates on arXiv.org
Google DeepMind News
Google DeepMind News
Security Latest
Security Latest
博客园 - 司徒正美
云风的 BLOG
云风的 BLOG
博客园 - 叶小钗
H
Hackread – Cybersecurity News, Data Breaches, AI and More
N
News and Events Feed by Topic
爱范儿
爱范儿
P
Proofpoint News Feed
C
CERT Recently Published Vulnerability Notes
Project Zero
Project Zero
Application and Cybersecurity Blog
Application and Cybersecurity Blog
Cisco Talos Blog
Cisco Talos Blog
GbyAI
GbyAI
cs.AI updates on arXiv.org
cs.AI updates on arXiv.org
Apple Machine Learning Research
Apple Machine Learning Research
T
Tenable Blog
Threat Intelligence Blog | Flashpoint
Threat Intelligence Blog | Flashpoint
V
Vulnerabilities – Threatpost
Forbes - Security
Forbes - Security
博客园 - 三生石上(FineUI控件)
C
Cyber Attacks, Cyber Crime and Cyber Security
N
News and Events Feed by Topic
V
V2EX
Webroot Blog
Webroot Blog
The Register - Security
The Register - Security
OSCHINA 社区最新新闻
OSCHINA 社区最新新闻
阮一峰的网络日志
阮一峰的网络日志
freeCodeCamp Programming Tutorials: Python, JavaScript, Git & More
Blog — PlanetScale
Blog — PlanetScale
M
MIT News - Artificial intelligence
Scott Helme
Scott Helme
Simon Willison's Weblog
Simon Willison's Weblog
L
LangChain Blog
W
WeLiveSecurity
Cloudbric
Cloudbric

博客园 - 我想我是青蛙

将golang程序注册为windows服务 CentOS上安装spark standalone mode(转载) 关于听歌这回事 c#读取apk 信息 golang 通用Contains方法 golang读取文件信息插入mongodb PetaPoco介绍 白话MongoDB(三)(转载) 白话MongoDB(二)(转载) 白话MongoDB(一) (转载) 给文章加入关键字链接 针对firefox ie6 ie7 ie8的css样式hack (转载) 好久不写日志了,现在开始,好好写了。。 sharepoint 查询calendar recurrence sharepoint之lookup字段 sharepoint获取Audiences 获取exchangeserve的calendar的item sharepoint错误处理 c#上传下载ftp(支持断点续传)
golang 读取mongob数据写入sqlserver
我想我是青蛙 · 2013-06-01 · via 博客园 - 我想我是青蛙

  今天有一个从mongodb读取数据,然后放到sqlserver的工作,当然这等程序必须用go来完成啊。

    先准备mongdb的第三方驱动包 http://labix.org/mgo

      odbc的第三方驱动包 https://bitbucket.org/miquella/mgodbc

  读取mongodb的数据的条件是根据日期范围,以及字符串条件。上代码。

package main

import (
    _ "bitbucket.org/miquella/mgodbc"
    "database/sql"
    "encoding/json"
    "fmt"
    "labix.org/v2/mgo"
    "labix.org/v2/mgo/bson"
    "os"
    "runtime"
    "time"
)

type serverslice struct {
    Servers []string
    Sqlconn string
    Start   string
    End     string
}

type user struct {
    UserName string    "UserName"
    Password string    "Password"
    Email    string    "Email"
    Phone    string    "Phone"
    SubTime  time.Time "SubTime"
}

var config serverslice
var worker = runtime.NumCPU()

//初始化配置
func init() {
    file, _ := os.Open("config.json")
    defer file.Close()
    buf := make([]byte, 2048)

    n, _ := file.Read(buf)
    err := json.Unmarshal(buf[:n], &config)
    if err != nil {
        panic(err)
        fmt.Println(err)
    }
}

func main() {
    runtime.GOMAXPROCS(runtime.NumCPU())

    var chanUser = make(chan user)
    // 标记完成
    dones := make(chan struct{}, worker)

    go readmongodb(chanUser)

    for i := 0; i < worker; i++ {
        go writesql(chanUser, dones)
    }
    awaitForCloseResult(dones)
    fmt.Println("完成")
}

//读取mongodb数据
func readmongodb(chanUser chan<- user) {
    start, _ := time.Parse(layout, config.Start)
    end, _ := time.Parse(layout, config.End)
    for _, server := range config.Servers {
        session, err := mgo.Dial(server)
        if err != nil {
            fmt.Println("打开", server, "失败")
            panic(err)
        }
        defer session.Close()
        var regex = bson.RegEx{}
        regex.Pattern = "^139.*"
        //查询条件是 start<=subtime<=end;email=nil;phone 不以139开头
        var query = bson.M{"SubTime": bson.M{"$gte": start, "$lte": end}, "Email": nil,
            "Phone": bson.M{"$not": regex}}

        //sqlreader类似
        iter := session.DB("db").C("Users").Find(query).Iter()
        message := user{}
        for iter.Next(&message) {
            chanUser <- message
        }
        //关闭通道
        close(chanUser)
    }
}

//写入sqlserver
func writesql(chanUser <-chan user, dones chan<- struct{}) {
    con, err := sql.Open("mgodbc", config.Sqlconn)
    if err != nil {
        fmt.Println(err)
        return
    }
    defer con.Close()
    //con.Exec(prepareSQL)
    for message := range chanUser {
        //组装sql
        var sql = fmt.Sprintf(insertSql, message.UserName, message.Password, message.Email,
            message.Phone, message.SubTime.Format(layout))

        _, err = con.Exec(sql)
        if err != nil {
            fmt.Println(err)
        }
    }
    dones <- struct{}{}
}

/*等待操作完成*/
func awaitForCloseResult(dones <-chan struct{}) {
    for {
        <-dones
        worker--
        if worker <= 0 {
            return
        }
    }
}

var layout = "2006-01-02 15:04:05"

var insertSql string = `INSERT INTO [dbo].[User]
                ([UserName]
                       ,[Password]
                       ,[Email]
                       ,[Phone]
                ,[SubTime] )
                 VALUES
                ('%s','%s','%s','%s','%s');
                `

对应的配置文件config.json

{
    "servers":
    [
        "127.0.0.1:27017"
    ],
    "sqlconn":"driver={SQL Server};SERVER=localhost;UID=aa;PWD=bbb;DATABASE=auth",
    "start":"2013-05-17 00:00:00",
    "end":"2013-06-01 00:00:00"
}