Kaynağa Gözat

init

master
Robin Thoni 8 yıl önce
işleme
2756dd5845
2 değiştirilmiş dosya ile 150 ekleme ve 0 silme
  1. 114
    0
      parallel_jobs.sh
  2. 36
    0
      parallel_jobs_example.sh

+ 114
- 0
parallel_jobs.sh Dosyayı Görüntüle

@@ -0,0 +1,114 @@
1
+#!/usr/bin/env bash
2
+
3
+launchThread()
4
+{
5
+    local threadFirst=${1}
6
+    local threadLast=${2}
7
+    local threadCount=${3}
8
+    local threadNumber=${4}
9
+    local parallelNbItems=${5}
10
+
11
+    local instanceNumber=0
12
+    for i in $(seq ${threadFirst} ${parallelNbItems} ${threadLast})
13
+    do
14
+        local instanceBegin=${i}
15
+        local instanceEnd=$((${i} + ${parallelNbItems} - 1))
16
+        if [ ${instanceEnd} -gt ${threadLast} ]
17
+        then
18
+            instanceEnd=${threadLast}
19
+        fi
20
+        local instanceCount=$((${instanceEnd} - ${instanceBegin} + 1))
21
+        echo "Launching thread ${threadNumber} instance ${instanceNumber}: ${instanceBegin} ${instanceEnd} ${instanceCount}"
22
+        launchInstance ${instanceBegin} ${instanceEnd} ${instanceCount} ${threadNumber} ${instanceNumber}
23
+        instanceNumber=$((${instanceNumber} + 1))
24
+    done
25
+}
26
+
27
+launch()
28
+{
29
+    local first=$1
30
+    local last=$2
31
+    local threads=$3
32
+    local parallelNbItems=$4
33
+    local itemsCount=$((${last} - ${first} + 1))
34
+
35
+    if [ ${threads} -gt ${itemsCount} ]
36
+    then
37
+        threads=${itemsCount}
38
+    fi
39
+
40
+    local parallelItems=$(( $((${itemsCount} + $((${itemsCount} % ${threads})))) / ${threads}))
41
+    if [ "${parallelNbItems}" == "" ] || [ ${parallelNbItems} -gt ${parallelItems} ]
42
+    then
43
+        parallelNbItems=${parallelItems}
44
+    fi
45
+
46
+    echo "Running pre launch"
47
+    preLaunch
48
+
49
+    echo "Processing ${itemsCount} elements in ${threads} threads with ${parallelNbItems} by ${parallelNbItems} elements per thread"
50
+
51
+
52
+    local threadsPid=()
53
+    local threadNumber=0
54
+    # trap ctrl_c INT
55
+    for i in $(seq ${first} ${parallelItems} ${last})
56
+    do
57
+        local threadFirst=${i}
58
+        local threadLast=$((${i} + ${parallelItems} - 1))
59
+        if [ ${threadLast} -gt ${last} ]
60
+        then
61
+            threadLast=${last}
62
+        fi
63
+        local threadCount=$((${threadLast} - ${threadFirst} + 1))
64
+        echo "Launching thread ${threadNumber}: from ${threadFirst} to ${threadLast} (${threadCount})"
65
+        launchThread ${threadFirst} ${threadLast} ${threadCount} ${threadNumber} ${parallelNbItems} &
66
+        local threadPid=$!
67
+        threadsPid[${threadNumber}]=${threadPid}
68
+        threadNumber=$((${threadNumber} + 1))
69
+    done
70
+
71
+    local stillRunningCount=${#threadsPid[@]}
72
+    local firstRun=1
73
+    while [ ${stillRunningCount} -ne 0 ]
74
+    do
75
+        stillRunningCount=0
76
+        local newFinished=0
77
+        for i in $(seq 0 $((${#threadsPid[@]} - 1)))
78
+        do
79
+            pid=${threadsPid[${i}]}
80
+            if [ "${pid}" == "" ]
81
+            then
82
+                pid=0
83
+            fi
84
+            if [ ${pid} -ne 0 ]
85
+            then
86
+                kill -0 ${pid} > /dev/null 2>/dev/null
87
+                local ret=$?
88
+                if [ ${ret} -eq 1 ]
89
+                then
90
+                    echo "Thread ${i} has finished"
91
+                    newFinished=1
92
+                    threadsPid[${i}]=0
93
+                else
94
+                    stillRunningCount=$((${stillRunningCount} + 1))
95
+                fi
96
+            fi
97
+        done
98
+        if [ ${stillRunningCount} -ne 0 ]
99
+        then
100
+            if [ ${newFinished} -eq 1 ] && [ ${stillRunningCount} -ne 0 ]
101
+            then
102
+                echo "${stillRunningCount} remaining threads"
103
+            fi
104
+            sleep 1
105
+        fi
106
+    done
107
+    echo "All threads done"
108
+
109
+
110
+    echo "Running post launch"
111
+    postLaunch
112
+}
113
+
114
+launch ${first} ${last} ${threads} "${parallelNbItems}"

+ 36
- 0
parallel_jobs_example.sh Dosyayı Görüntüle

@@ -0,0 +1,36 @@
1
+#! /usr/bin/env bash
2
+
3
+# first element to process, inclusive
4
+first=1
5
+# last element to process, inclusive
6
+last=2259
7
+# number of parallel instances to launch
8
+threads=100
9
+# max number of element to process per parallel instance, optional (automatically computed, all items processed at once)
10
+#parallelNbItems=10
11
+
12
+preLaunch()
13
+{
14
+    echo "Removing old data..."
15
+    sleep $((${RANDOM} % 10))
16
+}
17
+
18
+postLaunch()
19
+{
20
+    echo "Processing all data..."
21
+    sleep $((${RANDOM} % 10))
22
+}
23
+
24
+launchInstance()
25
+{
26
+    local instanceBegin=${1}
27
+    local instanceEnd=${2}
28
+    local instanceCount=${3}
29
+    local threadNumber=${4}
30
+    local instanceNumber=${5}
31
+    sleep $((${RANDOM} % 10))
32
+}
33
+
34
+#==============================================================================
35
+
36
+. ./parallel_jobs.sh

Loading…
İptal
Kaydet