2020年7月14日火曜日

thumbnail using goroutine stream

// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// +build ignore

// The thumbnail command produces thumbnails of JPEG files
// whose names are provided on each line of the standard input.
//
// The "+build ignore" tag (see p.295) excludes this file from the
// thumbnail package, but it can be compiled as a command and run like
// this:
//
// Run with:
//   $ go run $GOPATH/src/gopl.io/ch8/thumbnail/main.go
//   foo.jpeg
//   ^D
//
package main

import (
"bufio"
"fmt"
"log"
"os"

"gopl.io/ch8/thumbnail"
)

func main() {
input := bufio.NewScanner(os.Stdin)
for input.Scan() {
thumb, err := thumbnail.ImageFile(input.Text())
if err != nil {
log.Print(err)
continue
}
fmt.Println(thumb)
}
if err := input.Err(); err != nil {
log.Fatal(err)
}
}
// thumnail.go
// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 234.

// The thumbnail package produces thumbnail-size images from
// larger images.  Only JPEG images are currently supported.
package thumbnail

import (
"fmt"
"image"
"image/jpeg"
"io"
"os"
"path/filepath"
"strings"
)

// Image returns a thumbnail-size version of src.
func Image(src image.Image) image.Image {
// Compute thumbnail size, preserving aspect ratio.
xs := src.Bounds().Size().X
ys := src.Bounds().Size().Y
width, height := 128, 128
if aspect := float64(xs) / float64(ys); aspect < 1.0 {
width = int(128 * aspect) // portrait
} else {
height = int(128 / aspect) // landscape
}
xscale := float64(xs) / float64(width)
yscale := float64(ys) / float64(height)

dst := image.NewRGBA(image.Rect(0, 0, width, height))

// a very crude scaling algorithm
for x := 0; x < width; x++ {
for y := 0; y < height; y++ {
srcx := int(float64(x) * xscale)
srcy := int(float64(y) * yscale)
dst.Set(x, y, src.At(srcx, srcy))
}
}
return dst
}

// ImageStream reads an image from r and
// writes a thumbnail-size version of it to w.
func ImageStream(w io.Writer, r io.Reader) error {
src, _, err := image.Decode(r)
if err != nil {
return err
}
dst := Image(src)
return jpeg.Encode(w, dst, nil)
}

// ImageFile2 reads an image from infile and writes
// a thumbnail-size version of it to outfile.
func ImageFile2(outfile, infile string) (err error) {
in, err := os.Open(infile)
if err != nil {
return err
}
defer in.Close()

out, err := os.Create(outfile)
if err != nil {
return err
}

if err := ImageStream(out, in); err != nil {
out.Close()
return fmt.Errorf("scaling %s to %s: %s", infile, outfile, err)
}
return out.Close()
}

// ImageFile reads an image from infile and writes
// a thumbnail-size version of it in the same directory.
// It returns the generated file name, e.g. "foo.thumb.jpeg".
func ImageFile(infile string) (string, error) {
ext := filepath.Ext(infile) // e.g., ".jpg", ".JPEG"
outfile := strings.TrimSuffix(infile, ext) + ".thumb" + ext
return outfile, ImageFile2(outfile, infile)
}

// thum_test_test.go

// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// This file is just a place to put example code from the book.
// It does not actually run any code in gopl.io/ch8/thumbnail.

package thumbnail_test

import (
"log"
"os"
"sync"

"gopl.io/ch8/thumbnail"
)

//!+1
// makeThumbnails makes thumbnails of the specified files.
func makeThumbnails(filenames []string) {
for _, f := range filenames {
if _, err := thumbnail.ImageFile(f); err != nil {
log.Println(err)
}
}
}

//!-1

//!+2
// NOTE: incorrect!
func makeThumbnails2(filenames []string) {
for _, f := range filenames {
go thumbnail.ImageFile(f) // NOTE: ignoring errors
}
}

//!-2

//!+3
// makeThumbnails3 makes thumbnails of the specified files in parallel.
func makeThumbnails3(filenames []string) {
ch := make(chan struct{})
for _, f := range filenames {
go func(f string) {
thumbnail.ImageFile(f) // NOTE: ignoring errors
ch <- struct{}{}
}(f)
}

// Wait for goroutines to complete.
for range filenames {
<-ch
}
}

//!-3

//!+4
// makeThumbnails4 makes thumbnails for the specified files in parallel.
// It returns an error if any step failed.
func makeThumbnails4(filenames []string) error {
errors := make(chan error)

for _, f := range filenames {
go func(f string) {
_, err := thumbnail.ImageFile(f)
errors <- err
}(f)
}

for range filenames {
if err := <-errors; err != nil {
return err // NOTE: incorrect: goroutine leak!
}
}

return nil
}

//!-4

//!+5
// makeThumbnails5 makes thumbnails for the specified files in parallel.
// It returns the generated file names in an arbitrary order,
// or an error if any step failed.
func makeThumbnails5(filenames []string) (thumbfiles []string, err error) {
type item struct {
thumbfile string
err       error
}

ch := make(chan item, len(filenames))
for _, f := range filenames {
go func(f string) {
var it item
it.thumbfile, it.err = thumbnail.ImageFile(f)
ch <- it
}(f)
}

for range filenames {
it := <-ch
if it.err != nil {
return nil, it.err
}
thumbfiles = append(thumbfiles, it.thumbfile)
}

return thumbfiles, nil
}

//!-5

//!+6
// makeThumbnails6 makes thumbnails for each file received from the channel.
// It returns the number of bytes occupied by the files it creates.
func makeThumbnails6(filenames <-chan string) int64 {
sizes := make(chan int64)
var wg sync.WaitGroup // number of working goroutines
for f := range filenames {
wg.Add(1)
// worker
go func(f string) {
defer wg.Done()
thumb, err := thumbnail.ImageFile(f)
if err != nil {
log.Println(err)
return
}
info, _ := os.Stat(thumb) // OK to ignore error
sizes <- info.Size()
}(f)
}

// closer
go func() {
wg.Wait()
close(sizes)
}()

var total int64
for size := range sizes {
total += size
}
return total
}

//!-6
package main

import (
//"bufio"
//"fmt"
"log"
"os"
"sync"

"gopl.io/ch8/thumbnail"
)

func makeThumbnails6(filenames <-chan string) int64 {
sizes := make(chan int64)
var wg sync.WaitGroup // number of working goroutines
for f := range filenames {
wg.Add(1)
// worker
go func(f string) {
defer wg.Done()
thumb, err := thumbnail.ImageFile(f)
if err != nil {
log.Println(err)
return
}
info, _ := os.Stat(thumb) // OK to ignore error
sizes <- info.Size()
}(f)
}

// closer
go func() {
wg.Wait()
close(sizes)
}()

var total int64
for size := range sizes {
total += size
}
return total
}
----------------------------------------------------------------------  using stream
func main() {
files := []string{"monkey.jpeg","elephant.jpeg"}
generator := func(files []string) <-chan string {
fileStream := make(chan string)
go func() {
defer close(fileStream)
for _, f := range files {
fileStream <- f
}
}()
return fileStream
}
filenames := generator(files)

makeThumbnails6(filenames)
}
---------------------------------------------------------------------------- another using stream
package main

import (
//"bufio"
//"fmt"
//"log"
//"os"

"gopl.io/ch8/thumbnail"
)
func makeThumbnails5(filenames []string) (thumbfiles []string, err error) {
type item struct {
thumbfile string
err       error
}

ch := make(chan item, len(filenames))
for _, f := range filenames {
go func(f string) {
var it item
it.thumbfile, it.err = thumbnail.ImageFile(f)
ch <- it
}(f)
}

for range filenames {
it := <-ch
if it.err != nil {
return nil, it.err
}
thumbfiles = append(thumbfiles, it.thumbfile)
}

return thumbfiles, nil
}
func main() {
files := []string{"monkey.jpeg","elephant.jpeg"}
makeThumbnails5(files)
}

2020年7月5日日曜日

crawl3a and tee.go

 // Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 243.

// Crawl3 crawls web links starting with the command-line arguments.
//
// This version uses bounded parallelism.
// For simplicity, it does not address the termination problem.
//
package main

import (
"fmt"
"log"
"os"

"./links"
)

func crawl(url string) []string {
fmt.Println(url)
list, err := links.Extract(url)
if err != nil {
log.Print(err)
}
return list
}

//!+
func main() {
worklist := make(chan []string)  // lists of URLs, may have duplicates
unseenLinks := make(chan string) // de-duplicated URLs
    var n int
// Add command-line arguments to worklist.
go func() { worklist <- os.Args[1:] }()

// Create 20 crawler goroutines to fetch each unseen link.
for i := 0; i < 20; i++ {
go func() {
for link := range unseenLinks {
foundLinks := crawl(link)
go func() { worklist <- foundLinks }()
}
}()
}

// The main goroutine de-duplicates worklist items
// and sends the unseen ones to the crawlers.
seen := make(map[string]bool)
n++
for ; n > 0; n-- {
    list := <-worklist
for _, link := range list {
if !seen[link] {
seen[link] = true
n++
unseenLinks <- link
}
}
}
}

//!-
---------------------

ーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーーー
package main

import (
    "fmt"
)

func main() {

        repeat := func(
        done <-chan interface{},
        values ...interface{},
    ) <-chan interface{} {
        valueStream := make(chan interface{})
        go func() {
            defer close(valueStream)
            for {
                for _, v := range values {
                    select {
                    case <-done:
                        return
                    case valueStream <- v:
                    }
                }
            }
        }()
        return valueStream
    }
   
    take := func(
        done <-chan interface{},
        valueStream <-chan interface{},
        num int,
    ) <-chan interface{} {
        takeStream := make(chan interface{})
        go func() {
            defer close(takeStream)
            for i := 0; i < num; i++ {
                select {
                case <-done:
                    return
                case takeStream <- <-valueStream:
                }
            }
        }()
        return takeStream
    }
   
    orDone := func(done, c <-chan interface{}) <-chan interface{} {
        valStream := make(chan interface{})
        go func() {
            defer close(valStream)
            for {
                select {
                case <-done:
                    return
                case v, ok := <-c:
                    if ok == false {
                        return
                    }
                    select {
                    case valStream <- v:
                    case <-done:
                    }
                }
            }
        }()
        return valStream
    }
   
    tee := func(
      done <- chan interface{},
      in <- chan interface{},
      ) (_, _ <- chan interface{}) {
       out1 := make(chan interface{})
       out2 := make(chan interface{})
       go func() {
         defer close(out1)
         defer close(out2)
         for val := range orDone(done, in) {
          var out1, out2 = out1, out2
          for i := 0; i < 2; i++ {
            select {
              case out1<-val:
                out1 = nil
              case out2<-val:
                out2 = nil
            }
          }
       }
      }()
      return out1, out2
    }
   
    done := make(chan interface{})
    defer close(done)
   
    out1, out2 := tee(done, take(done, repeat(done, 1, 2), 4))
   
    for val1 := range out1 {
      fmt.Printf("out1: %v, out2: %v\n", val1, <-out2)
    }
   
}