Go 并发爬虫 — goroutine + channel 实战
目标
演示 Go 核心卖点:goroutine 轻量并发 + channel 通信。编写一个并发检查多个 URL 响应状态的工具。
完整代码
package main
import (
"fmt"
"net/http"
"sync"
"time"
)
type URLResult struct {
URL string
StatusCode int
Duration time.Duration
Error error
}
func checkURL(url string, resultCh chan<- URLResult) {
start := time.Now()
resp, err := http.Get(url)
duration := time.Since(start)
if err != nil {
resultCh <- URLResult{URL: url, Duration: duration, Error: err}
return
}
defer resp.Body.Close()
resultCh <- URLResult{URL: url, StatusCode: resp.StatusCode, Duration: duration}
}
func main() {
urls := []string{
"https://www.baidu.com",
"https://www.github.com",
"https://www.google.com",
"https://httpbin.org/delay/2",
"https://www.zhihu.com",
}
resultCh := make(chan URLResult, len(urls))
var wg sync.WaitGroup
for _, url := range urls {
wg.Add(1)
go func(u string) {
defer wg.Done()
checkURL(u, resultCh)
}(url)
}
go func() {
wg.Wait()
close(resultCh)
}()
fmt.Println("🌐 URL 健康检查结果:")
fmt.Println("-----------------------------")
successCount, failCount := 0, 0
for result := range resultCh {
if result.Error != nil {
fmt.Printf("❌ %-35s | 错误: %v\n", result.URL, result.Error)
failCount++
} else {
fmt.Printf("✅ %-35s | %3d | %v\n", result.URL, result.StatusCode, result.Duration)
successCount++
}
}
fmt.Println("-----------------------------")
fmt.Printf("成功: %d 失败: %d 总计: %d\n", successCount, failCount, len(urls))
}
运行步骤
go mod init healthcheck
go run main.go
预期输出
🌐 URL 健康检查结果:
-----------------------------
✅ https:
✅ https:
❌ https:
✅ https:
✅ https:
-----------------------------
成功: 4 失败: 1 总计: 5
关键知识点
go func() 创建 goroutine,开销仅 2KB 栈
chan 安全地在 goroutine 间传递数据
sync.WaitGroup 等待所有 goroutine 完成
- 所有 URL 并发检测,总耗时约等于最慢的单个请求
Go JWT 认证中间件
目标
用 golang-jwt/jwt 实现 Token 签发和验证,作为 Gin 中间件保护私有 API。
完整代码
package main
import (
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/golang-jwt/jwt/v5"
)
var jwtSecret = []byte("your-secret-key-change-in-production")
type Claims struct {
UserID int `json:"userId"`
Username string `json:"username"`
Role string `json:"role"`
jwt.RegisteredClaims
}
func GenerateToken(userID int, username, role string) (string, error) {
claims := Claims{
UserID: userID,
Username: username,
Role: role,
RegisteredClaims: jwt.RegisteredClaims{
ExpiresAt: jwt.NewNumericDate(time.Now().Add(7 * 24 * time.Hour)),
IssuedAt: jwt.NewNumericDate(time.Now()),
Issuer: "my-bishe",
},
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
return token.SignedString(jwtSecret)
}
func ParseToken(tokenStr string) (*Claims, error) {
token, err := jwt.ParseWithClaims(tokenStr, &Claims{}, func(t *jwt.Token) (any, error) {
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, errors.New("无效签名算法")
}
return jwtSecret, nil
})
if err != nil {
return nil, err
}
claims, ok := token.Claims.(*Claims)
if !ok || !token.Valid {
return nil, errors.New("无效 Token")
}
return claims, nil
}
func AuthMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
tokenStr := c.GetHeader("Authorization")
if len(tokenStr) < 8 || tokenStr[:7] != "Bearer " {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "未提供 Token"})
return
}
claims, err := ParseToken(tokenStr[7:])
if err != nil {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Token 无效或已过期"})
return
}
c.Set("userId", claims.UserID)
c.Set("username", claims.Username)
c.Set("role", claims.Role)
c.Next()
}
}
func RoleMiddleware(roles ...string) gin.HandlerFunc {
return func(c *gin.Context) {
role, _ := c.Get("role")
for _, r := range roles {
if role == r {
c.Next()
return
}
}
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "无权访问"})
}
}
func main() {
r := gin.Default()
r.POST("/login", func(c *gin.Context) {
var req struct {
Username string `json:"username"`
Password string `json:"password"`
}
c.ShouldBindJSON(&req)
if req.Username == "admin" && req.Password == "123456" {
token, _ := GenerateToken(1, req.Username, "admin")
c.JSON(200, gin.H{"token": token, "expiresIn": "7天"})
} else {
c.JSON(401, gin.H{"error": "账号或密码错误"})
}
})
auth := r.Group("/api").Use(AuthMiddleware())
{
auth.GET("/profile", func(c *gin.Context) {
c.JSON(200, gin.H{
"userId": c.GetInt("userId"),
"username": c.GetString("username"),
"role": c.GetString("role"),
})
})
admin := auth.Group("/admin").Use(RoleMiddleware("admin"))
{
admin.GET("/dashboard", func(c *gin.Context) {
c.JSON(200, gin.H{"message": "管理员面板"})
})
}
}
r.Run(":8080")
}
测试
curl -X POST http:
-H "Content-Type: application/json" \
-d '{"username":"admin","password":"123456"}'
curl http:
-H "Authorization: Bearer <TOKEN>"
关键要点
- HS256 对称加密,简单够用;生产环境用 RS256(非对称)
- Token 存在客户端,服务端不存储状态
- 过期时间设置 7 天,可选实现 Refresh Token 续期