crawl3a and tee.go

 // Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.

// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 243.

// Crawl3 crawls web links starting with the command-line arguments.

//

// This version uses bounded parallelism.

// For simplicity, it does not address the termination problem.

//

package main

import (

"fmt"

"log"

"os"

"./links"

)

func crawl(url string) []string {

fmt.Println(url)

list, err := links.Extract(url)

if err != nil {

log.Print(err)

}

return list

}

//!+

func main() {

worklist := make(chan []string)  // lists of URLs, may have duplicates

unseenLinks := make(chan string) // de-duplicated URLs

    var n int

// Add command-line arguments to worklist.

go func() { worklist <- os.Args[1:] }()

// Create 20 crawler goroutines to fetch each unseen link.

for i := 0; i < 20; i++ {

go func() {

for link := range unseenLinks {

foundLinks := crawl(link)

go func() { worklist <- foundLinks }()

}

}()

}

// The main goroutine de-duplicates worklist items

// and sends the unseen ones to the crawlers.

seen := make(map[string]bool)

n++

for ; n > 0; n-- {

    list := <-worklist

for _, link := range list {

if !seen[link] {

seen[link] = true

n++

unseenLinks <- link

}

}

}

}

//!-

---------------------

ーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーー
package main

import (
    "fmt"
)

func main() {

        repeat := func(
        done <-chan interface{},
        values ...interface{},
    ) <-chan interface{} {
        valueStream := make(chan interface{})
        go func() {
            defer close(valueStream)
            for {
                for _, v := range values {
                    select {
                    case <-done:
                        return
                    case valueStream <- v:
                    }
                }
            }
        }()
        return valueStream
    }
   
    take := func(
        done <-chan interface{},
        valueStream <-chan interface{},
        num int,
    ) <-chan interface{} {
        takeStream := make(chan interface{})
        go func() {
            defer close(takeStream)
            for i := 0; i < num; i++ {
                select {
                case <-done:
                    return
                case takeStream <- <-valueStream:
                }
            }
        }()
        return takeStream
    }
   
    orDone := func(done, c <-chan interface{}) <-chan interface{} {
        valStream := make(chan interface{})
        go func() {
            defer close(valStream)
            for {
                select {
                case <-done:
                    return
                case v, ok := <-c:
                    if ok == false {
                        return
                    }
                    select {
                    case valStream <- v:
                    case <-done:
                    }
                }
            }
        }()
        return valStream
    }
   
    tee := func(
      done <- chan interface{},
      in <- chan interface{},
      ) (_, _ <- chan interface{}) {
       out1 := make(chan interface{})
       out2 := make(chan interface{})
       go func() {
         defer close(out1)
         defer close(out2)
         for val := range orDone(done, in) {
          var out1, out2 = out1, out2
          for i := 0; i < 2; i++ {
            select {
              case out1<-val:
                out1 = nil
              case out2<-val:
                out2 = nil
            }
          }
       }
      }()
      return out1, out2
    }
   
    done := make(chan interface{})
    defer close(done)
   
    out1, out2 := tee(done, take(done, repeat(done, 1, 2), 4))
   
    for val1 := range out1 {
      fmt.Printf("out1: %v, out2: %v\n", val1, <-out2)
    }
   
}