argo-controller-crash-log.txt

time="2020-07-28T16:08:04Z" level=info msg="Alloc=9702 TotalAlloc=2322341 Sys=71872 NumGC=739 Goroutines=165"

time="2020-07-28T16:08:53Z" level=info msg="Processing default/foo-workflow"


goroutine 139 [running]:

k8s.io/apimachinery/pkg/util/runtime.logPanic(0x171b000, 0xc0012ea520)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:74 +0xa3

k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:48 +0x82

panic(0x171b000, 0xc0012ea520)

	/usr/local/go/src/runtime/panic.go:679 +0x1b2

github.com/valyala/fasttemplate.New(...)

	/go/pkg/mod/github.com/valyala/[email protected]/template.go:138

github.com/argoproj/argo/workflow/common.SubstituteParams(0xc00053e480, 0xc0006f0a50, 0xc0006f0f90, 0x28ad5e0, 0x5, 0x5)

	/go/src/github.com/argoproj/argo/workflow/common/util.go:310 +0xdb2

github.com/argoproj/argo/workflow/common.ProcessArgs(0xc00053e240, 0x1bb8e00, 0x28ad5e0, 0xc0006f0a50, 0xc0006f0f90, 0xc00118e901, 0x11, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/common/util.go:299 +0x354

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc00051f880, 0xc00053e240, 0xc000f2a480, 0x1bb8e00, 0x28ad5e0, 0xc0004f9b30, 0x18d7f00, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:407 +0x2e9

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00051f880, 0x1bb8900, 0xc00141e620, 0xc000f2a1c0, 0x1bb8e00, 0x28ad5e0, 0xc0004f9b30, 0x12e, 0xe, 0x400)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateDAG(0xc00051f880, 0xc0006f0c60, 0xc000f2a1c0, 0xc000324480, 0xc0014890b8, 0xc00118e900)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:1185 +0x4a5

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc00051f880, 0xc0003a4480, 0xc000f2a1c0, 0x1bb87c0, 0xc0006f0b10, 0xc0004fabf0, 0x18a4d00, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:427 +0xc40

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00051f880, 0x1bb8a40, 0xc0016784d0, 0xc0004fad80, 0x1bb87c0, 0xc0006f0b10, 0xc0004fabf0, 0x0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169

github.com/argoproj/argo/workflow/validate.ValidateWorkflow(0x1bab600, 0xc000f49d20, 0x1bab5e0, 0xc000f49d40, 0xc00124f800, 0xc001758b00, 0x0, 0x0, 0x0, 0xc000f49d30, ...)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:234 +0x13c8

github.com/argoproj/argo/workflow/validate.ValidateCronWorkflow(0x1bab600, 0xc000f49d20, 0x1bab5e0, 0xc000f49d40, 0xc0004a6700, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:317 +0x1a8

github.com/argoproj/argo/workflow/cron.(*cronWfOperationCtx).validateCronWorkflow(0xc000182af0, 0x1bb8b00, 0xc0003fd800)

	/go/src/github.com/argoproj/argo/workflow/cron/operator.go:96 +0x178

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00079a180, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:152 +0x438

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00079a180)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00083c2e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00083c2e0, 0x3b9aca00, 0x0, 0x1, 0xc000570480)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc00083c2e0, 0x3b9aca00, 0xc000570480)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc

panic: Cannot find end tag="}}" in the template="{\"name\":\"extract-data\",\"arguments\":{},\"inputs\":{},\"outputs\":{},\"metadata\":{},\"script\":{\"name\":\"\",\"image\":\"eu.gcr.io/jl-container-images/findability-insights/jl-recs-user-browse-training-workflow:latest\",\"args\":[\"--main extract\",\"--triggerDate {{arguments.parameters.triggerDate}\"],\"resources\":{},\"volumeMounts\":[{\"name\":\"workdir\",\"mountPath\":\"/data\"}],\"source\":\"\"},\"serviceAccountName\":\"pdp-ksa\"}" starting from "arguments.parameters.triggerDate}\"],\"resources\":{},\"volumeMounts\":[{\"name\":\"workdir\",\"mountPath\":\"/data\"}],\"source\":\"\"},\"serviceAccountName\":\"pdp-ksa\"}" [recovered]

	panic: Cannot find end tag="}}" in the template="{\"name\":\"extract-data\",\"arguments\":{},\"inputs\":{},\"outputs\":{},\"metadata\":{},\"script\":{\"name\":\"\",\"image\":\"eu.gcr.io/jl-container-images/findability-insights/jl-recs-user-browse-training-workflow:latest\",\"args\":[\"--main extract\",\"--triggerDate {{arguments.parameters.triggerDate}\"],\"resources\":{},\"volumeMounts\":[{\"name\":\"workdir\",\"mountPath\":\"/data\"}],\"source\":\"\"},\"serviceAccountName\":\"pdp-ksa\"}" starting from "arguments.parameters.triggerDate}\"],\"resources\":{},\"volumeMounts\":[{\"name\":\"workdir\",\"mountPath\":\"/data\"}],\"source\":\"\"},\"serviceAccountName\":\"pdp-ksa\"}"

goroutine 139 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:55 +0x105
panic(0x171b000, 0xc0012ea520)
	/usr/local/go/src/runtime/panic.go:679 +0x1b2
github.com/valyala/fasttemplate.New(...)
	/go/pkg/mod/github.com/valyala/[email protected]/template.go:138
github.com/argoproj/argo/workflow/common.SubstituteParams(0xc00053e480, 0xc0006f0a50, 0xc0006f0f90, 0x28ad5e0, 0x5, 0x5)
	/go/src/github.com/argoproj/argo/workflow/common/util.go:310 +0xdb2
github.com/argoproj/argo/workflow/common.ProcessArgs(0xc00053e240, 0x1bb8e00, 0x28ad5e0, 0xc0006f0a50, 0xc0006f0f90, 0xc00118e901, 0x11, 0x0, 0x0)
	/go/src/github.com/argoproj/argo/workflow/common/util.go:299 +0x354
github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc00051f880, 0xc00053e240, 0xc000f2a480, 0x1bb8e00, 0x28ad5e0, 0xc0004f9b30, 0x18d7f00, 0x0)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:407 +0x2e9
github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00051f880, 0x1bb8900, 0xc00141e620, 0xc000f2a1c0, 0x1bb8e00, 0x28ad5e0, 0xc0004f9b30, 0x12e, 0xe, 0x400)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169
github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateDAG(0xc00051f880, 0xc0006f0c60, 0xc000f2a1c0, 0xc000324480, 0xc0014890b8, 0xc00118e900)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:1185 +0x4a5
github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc00051f880, 0xc0003a4480, 0xc000f2a1c0, 0x1bb87c0, 0xc0006f0b10, 0xc0004fabf0, 0x18a4d00, 0x0)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:427 +0xc40
github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00051f880, 0x1bb8a40, 0xc0016784d0, 0xc0004fad80, 0x1bb87c0, 0xc0006f0b10, 0xc0004fabf0, 0x0, 0x0, 0x0)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169
github.com/argoproj/argo/workflow/validate.ValidateWorkflow(0x1bab600, 0xc000f49d20, 0x1bab5e0, 0xc000f49d40, 0xc00124f800, 0xc001758b00, 0x0, 0x0, 0x0, 0xc000f49d30, ...)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:234 +0x13c8
github.com/argoproj/argo/workflow/validate.ValidateCronWorkflow(0x1bab600, 0xc000f49d20, 0x1bab5e0, 0xc000f49d40, 0xc0004a6700, 0x0, 0x0)
	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:317 +0x1a8
github.com/argoproj/argo/workflow/cron.(*cronWfOperationCtx).validateCronWorkflow(0xc000182af0, 0x1bb8b00, 0xc0003fd800)
	/go/src/github.com/argoproj/argo/workflow/cron/operator.go:96 +0x178
github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00079a180, 0x0)
	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:152 +0x438
github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00079a180)
	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00083c2e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00083c2e0, 0x3b9aca00, 0x0, 0x1, 0xc000570480)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(0xc00083c2e0, 0x3b9aca00, 0xc000570480)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d
created by github.com/argoproj/argo/workflow/cron.(*Controller).Run
	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc









time="2020-07-28T16:09:09Z" level=info msg="config map" name=workflow-controller-configmap

time="2020-07-28T16:09:09Z" level=info msg="Configuration:\nartifactRepository: {}\nmetricsConfig: {}\nnodeEvents: {}\npodSpecLogStrategy: {}\nsso:\n  clientId:\n    key: \"\"\n  clientSecret:\n    key: \"\"\n  issuer: \"\"\n  redirectUrl: \"\"\ntelemetryConfig: {}\n"

time="2020-07-28T16:09:09Z" level=info msg="Persistence configuration disabled"

time="2020-07-28T16:09:09Z" level=info msg="Starting Workflow Controller" version=v2.9.1

time="2020-07-28T16:09:09Z" level=info msg="Workers: workflow: 32, pod: 32"

time="2020-07-28T16:09:09Z" level=info msg="Performing periodic GC every 5m0s"

time="2020-07-28T16:09:09Z" level=info msg="Starting CronWorkflow controller"

time="2020-07-28T16:09:09Z" level=info msg="Starting prometheus metrics server at localhost:9090/metrics"

time="2020-07-28T16:09:09Z" level=info msg="Starting workflow TTL controller (resync 20m0s)"

time="2020-07-28T16:09:09Z" level=info msg="Persistence disabled - so archived workflow GC disabled - you must restart the controller if you enable this"

time="2020-07-28T16:09:09Z" level=info msg="Processing release/nvals-categories-category-hierarchy-workflow"

time="2020-07-28T16:09:09Z" level=info msg="Processing default/disco-2801-workflow"

fatal error: concurrent map read and map write



goroutine 193 [running]:

runtime.throw(0x197342d, 0x21)

	/usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc00074fbc8 sp=0xc00074fb98 pc=0x42e5d2

runtime.mapaccess1_faststr(0x170e600, 0xc0006aeab0, 0xc000b04320, 0x19, 0x194d89b)

	/usr/local/go/src/runtime/map_faststr.go:21 +0x44f fp=0xc00074fc38 sp=0xc00074fbc8 pc=0x41285f

github.com/argoproj/argo/workflow/metrics.(*Metrics).WorkflowAdded(0xc0005fe240, 0xc000b04320, 0x19, 0xc00110db55, 0x9)

	/go/src/github.com/argoproj/argo/workflow/metrics/metrics.go:92 +0x56 fp=0xc00074fc78 sp=0xc00074fc38 pc=0x1495f56

github.com/argoproj/argo/workflow/controller.(*WorkflowController).addWorkflowInformerHandlers.func5(0x193f940, 0xc0003aa0f0)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:592 +0xbd fp=0xc00074fcd0 sp=0xc00074fc78 pc=0x15b4e6d

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd(0xc0006a9680, 0x193f940, 0xc0003aa0f0)

	<autogenerated>:1 +0x5a fp=0xc00074fcf0 sp=0xc00074fcd0 pc=0x130b5da

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xc00082ad90, 0x430340)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218 fp=0xc00074fd68 sp=0xc00074fcf0 pc=0x13093d8

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00074fdd8, 0xc00082ae20, 0x41eb4d)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51 fp=0xc00074fd90 sp=0xc00074fd68 pc=0x12c84b1

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79 fp=0xc00074fdf8 sp=0xc00074fd90 pc=0x1309499

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082af40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e fp=0xc00074fe68 sp=0xc00074fdf8 pc=0x12c8b6e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00074ff40, 0xdf8475800, 0x0, 0x1, 0xc000770600)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8 fp=0xc00074ff18 sp=0xc00074fe68 pc=0x12c8108

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000565580)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b fp=0xc00074ff68 sp=0xc00074ff18 pc=0x13035db

k8s.io/client-go/tools/cache.(*processorListener).run-fm()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:593 +0x2a fp=0xc00074ff80 sp=0xc00074ff68 pc=0x130b2da

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc000810590)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59 fp=0xc00074ffd0 sp=0xc00074ff80 pc=0x12c8ae9

runtime.goexit()

	/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00074ffd8 sp=0xc00074ffd0 pc=0x45b741

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 1 [select (no cases)]:

main.NewRootCommand.func1(0xc00016b180, 0xc000593100, 0x0, 0x4, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:89 +0x3f4

github.com/spf13/cobra.(*Command).execute(0xc00016b180, 0xc00003a060, 0x4, 0x4, 0xc00016b180, 0xc00003a060)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:762 +0x460

github.com/spf13/cobra.(*Command).ExecuteC(0xc00016b180, 0xc00006a750, 0xc00052bf50, 0x40576f)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:852 +0x2ea

github.com/spf13/cobra.(*Command).Execute(...)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:800

main.main()

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:111 +0x2b



goroutine 6 [chan receive]:

k8s.io/klog.(*loggingT).flushDaemon(0x288fb80)

	/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b

created by k8s.io/klog.init.0

	/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd6



goroutine 7 [syscall]:

os/signal.signal_recv(0x0)

	/usr/local/go/src/runtime/sigqueue.go:147 +0x9c

os/signal.loop()

	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22

created by os/signal.init.0

	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41



goroutine 50 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00080c0d8, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00080c0c8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00080c0b0, 0xc000810060, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000812080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007fee00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0007fee00, 0x3b9aca00, 0x0, 0xc000782001, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000812080, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000792000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:150 +0x52c



goroutine 11 [chan receive]:

github.com/argoproj/pkg/stats.RegisterStackDumper.func1()

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:31 +0xa0

created by github.com/argoproj/pkg/stats.RegisterStackDumper

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:27 +0x35



goroutine 13 [chan receive]:

github.com/argoproj/pkg/stats.StartStatsTicker.func1(0xc00003a550)

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:19 +0x37

created by github.com/argoproj/pkg/stats.StartStatsTicker

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:17 +0x4d



goroutine 14 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0003f4b40)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 15 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc00012df80)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 16 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0003f4c60)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 97 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000144000)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 26 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc00057e080, 0xc000750d18, 0xc0007ba000, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc000750d18, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc0005a25a0, 0xc000750e78, 0x1, 0x1, 0xc000345a40)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run(0xc000475880, 0x1bef540, 0xc000345a40, 0x20, 0x20)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:163 +0x8f3

created by main.NewRootCommand.func1

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:86 +0x3ef



goroutine 27 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00016ed38, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00016ed28)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00016ed10, 0xc00038e690, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0004abf80)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005b4ec8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005b4ec8, 0x3b9aca00, 0x0, 0xc0003abe01, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc0004abf80, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

github.com/argoproj/argo/config.(*controller).Run(0xc000341ec0, 0xc0005a25a0, 0xc0006ad3e0)

	/go/src/github.com/argoproj/argo/config/controller.go:119 +0x384

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:148 +0x44c



goroutine 25 [runnable]:

internal/poll.runtime_pollWait(0x7f9a4d6f4fd0, 0x72, 0xffffffffffffffff)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc0004abb98, 0x72, 0xcc00, 0xcc34, 0xffffffffffffffff)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Read(0xc0004abb80, 0xc00072a000, 0xcc34, 0xcc34, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf

net.(*netFD).Read(0xc0004abb80, 0xc00072a000, 0xcc34, 0xcc34, 0x203000, 0x421305, 0xc00000c060)

	/usr/local/go/src/net/fd_unix.go:202 +0x4f

net.(*conn).Read(0xc000544018, 0xc00072a000, 0xcc34, 0xcc34, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/net.go:184 +0x68

crypto/tls.(*atLeastReader).Read(0xc00000c060, 0xc00072a000, 0xcc34, 0xcc34, 0x415628, 0x7f9a4d7d6200, 0xc0008e1970)

	/usr/local/go/src/crypto/tls/conn.go:780 +0x60

bytes.(*Buffer).ReadFrom(0xc000096cd8, 0x1bab100, 0xc00000c060, 0x409eb5, 0x1755ec0, 0x18e2f80)

	/usr/local/go/src/bytes/buffer.go:204 +0xb4

crypto/tls.(*Conn).readFromUntil(0xc000096a80, 0x1bacec0, 0xc000544018, 0x5, 0xc000544018, 0x7f9a4d4f1570)

	/usr/local/go/src/crypto/tls/conn.go:802 +0xec

crypto/tls.(*Conn).readRecordOrCCS(0xc000096a80, 0x0, 0x0, 0xc000cc60f0)

	/usr/local/go/src/crypto/tls/conn.go:609 +0x124

crypto/tls.(*Conn).readRecord(...)

	/usr/local/go/src/crypto/tls/conn.go:577

crypto/tls.(*Conn).Read(0xc000096a80, 0xc0005db000, 0x1000, 0x1000, 0x0, 0x0, 0x0)

	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161

bufio.(*Reader).Read(0xc000542d80, 0xc0005dc038, 0x9, 0x9, 0xc000543aa0, 0xc0008e1dd8, 0xc0008e1cf0)

	/usr/local/go/src/bufio/bufio.go:226 +0x26a

io.ReadAtLeast(0x1baaee0, 0xc000542d80, 0xc0005dc038, 0x9, 0x9, 0x9, 0x404d15, 0xc0007fa6c0, 0xc0008e1dd8)

	/usr/local/go/src/io/io.go:310 +0x87

io.ReadFull(...)

	/usr/local/go/src/io/io.go:329

golang.org/x/net/http2.readFrameHeader(0xc0005dc038, 0x9, 0x9, 0x1baaee0, 0xc000542d80, 0x0, 0x0, 0xc000cc6000, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x87

golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005dc000, 0xc000cc6030, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0xa1

golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0008e1fb8, 0xc000467fa8, 0x88bb98)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1746 +0x8c

golang.org/x/net/http2.(*ClientConn).readLoop(0xc0002d0c00)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1674 +0xa3

created by golang.org/x/net/http2.(*Transport).newClientConn

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:674 +0x62f



goroutine 101 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc0004abf80)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 28 [runnable]:

sync.(*RWMutex).Unlock(0xc000768000)

	/usr/local/go/src/sync/rwmutex.go:118 +0xd2

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000768000, 0xc00076c010, 0x1757640, 0xc000350740, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:429 +0x2c3

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00076e000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00074ce00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00074ce00, 0x3b9aca00, 0x0, 0xc0000e6101, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00076e000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00067db00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:149 +0x4a2



goroutine 102 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000287b80, 0x1bbe340, 0xc00090a880, 0xc000889c00, 0xc00029e3c0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000287b80, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00054c6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000889ef0, 0x3b9aca00, 0x0, 0x409b01, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000287b80, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000bf430, 0xc000437fc0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 104 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0003f5620)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 198 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc000287b80, 0xc0005a25a0, 0xc0001227e0, 0xc00029e3c0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 114 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0002a2e00, 0xc000770000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000772000, 0xc000776040)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 115 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc00076e000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 116 [select]:

golang.org/x/net/http2.(*ClientConn).roundTrip(0xc0002d0c00, 0xc0007f6e00, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1096 +0x9a0

golang.org/x/net/http2.(*Transport).RoundTripOpt(0xc00029eba0, 0xc0007f6e00, 0xc0005e8d00, 0x754025, 0x0, 0xc0001a10e0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:448 +0x172

golang.org/x/net/http2.(*Transport).RoundTrip(...)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:409

golang.org/x/net/http2.noDialH2RoundTripper.RoundTrip(0xc00029eba0, 0xc0007f6e00, 0xc00086e160, 0x5, 0xc0001a1168)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2613 +0x3e

net/http.(*Transport).roundTrip(0xc0001bea00, 0xc0007f6e00, 0x16c7600, 0xc0005bce01, 0xc00113d600)

	/usr/local/go/src/net/http/transport.go:485 +0xdc1

net/http.(*Transport).RoundTrip(0xc0001bea00, 0xc0007f6e00, 0x1954ebc, 0xd, 0xc00113d6b8)

	/usr/local/go/src/net/http/roundtrip.go:17 +0x35

k8s.io/client-go/transport.(*bearerAuthRoundTripper).RoundTrip(0xc0006aed20, 0xc0007f6d00, 0x1951e42, 0xa, 0xc00113d540)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:317 +0x266

k8s.io/client-go/transport.(*userAgentRoundTripper).RoundTrip(0xc0006a9560, 0xc0007f6c00, 0xc0006a9560, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:167 +0x1c0

net/http.send(0xc0007f6c00, 0x1bacd40, 0xc0006a9560, 0x0, 0x0, 0x0, 0xc0008cc078, 0x5c8036, 0x1, 0x0)

	/usr/local/go/src/net/http/client.go:250 +0x443

net/http.(*Client).send(0xc0006aed50, 0xc0007f6c00, 0x0, 0x0, 0x0, 0xc0008cc078, 0x0, 0x1, 0xc000b0bb60)

	/usr/local/go/src/net/http/client.go:174 +0xfa

net/http.(*Client).do(0xc0006aed50, 0xc0007f6c00, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/http/client.go:641 +0x3ce

net/http.(*Client).Do(0xc0006aed50, 0xc0007f6c00, 0x0, 0x3, 0xc00086e160)

	/usr/local/go/src/net/http/client.go:509 +0x35

k8s.io/client-go/rest.(*Request).WatchWithSpecificDecoders(0xc0007e1380, 0xc0007cd7c8, 0x7f9a4d70b178, 0x28ad300, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:573 +0x1e2

k8s.io/client-go/dynamic.(*dynamicResourceClient).Watch(0xc0007c42d0, 0x0, 0x0, 0x0, 0x0, 0xc0000b5440, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:309 +0x8d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000bac8a8, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:39 +0x1e9

k8s.io/client-go/tools/cache.(*ListWatch).Watch(0xc0006a95e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:113 +0x6c

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000794000, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:275 +0x420

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00077f6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0007cdef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000794000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000772020, 0xc000776080)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 51 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00080c028, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00080c018)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00080c000, 0xc000810010, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000812000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007fce00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0007fce00, 0x3b9aca00, 0x0, 0xc00037c401, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000812000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00067db90, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:151 +0x582



goroutine 52 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podLabeler(0xc000475880, 0xc0005a25a0)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:215 +0xda

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:152 +0x5cb



goroutine 53 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podGarbageCollector(0xc000475880, 0xc0005a25a0)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:241 +0xd4

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:153 +0x614



goroutine 54 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).workflowGarbageCollector(0xc000475880, 0xc0005a25a0)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:275 +0x18e

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:154 +0x65d



goroutine 147 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000144380, 0xc0005a2600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0006ce510, 0xc000834500)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 56 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc000566220, 0xc000896e40, 0xc0000aaba0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc00088de40, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc0005a25a0, 0xc00088df28, 0x1, 0x1, 0xc000820f50)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run(0xc0000e5180, 0xc0005a25a0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:63 +0x211

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runTTLController(0xc000475880, 0x1bef540, 0xc000345a40)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:123 +0x86

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:157 +0x6e5



goroutine 57 [chan receive]:

github.com/argoproj/argo/workflow/cron.(*Controller).Run(0xc00039b380, 0x1bef540, 0xc000345a40)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:112 +0x66a

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runCronController(0xc000475880, 0x1bef540, 0xc000345a40)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:131 +0xc0

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:158 +0x724



goroutine 148 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc000565880)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 172 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc000794000, 0xc0005a25a0, 0xc0000abc80, 0xc0003f4660)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 59 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc0005a25a0, 0xc00079c010, 0x1bef540, 0xc0007a0000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 60 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc0007ba0c0, 0x5f5e100, 0x0, 0xc0007ba060)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 66 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0005b6dd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007a6740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005b6f40, 0xdf8475800, 0x0, 0x1, 0xc0000aa600)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000565500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0007f8000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 67 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000565500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0007f8010)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 68 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0005b7dd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007a7740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005b7f40, 0xdf8475800, 0x0, 0x1, 0xc0000aa660)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000565580)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0007f8020)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 69 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000565580)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0007f8030)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 105 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc0001441c0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 31 [chan receive]:

github.com/argoproj/argo/workflow/metrics.runServer(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc000586be0, 0x1bef540, 0xc000345a40)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:59 +0x291

created by github.com/argoproj/argo/workflow/metrics.(*Metrics).RunServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:36 +0x28f



goroutine 139 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000144460, 0xc000814240)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000822120, 0xc000824320)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 106 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0003f5740)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 107 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000144230)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 130 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0002a2ee0, 0xc000814000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000822000, 0xc000824040)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 131 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00081add8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082ef40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081af40, 0xdf8475800, 0x0, 0x1, 0xc0008141e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000565600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2f40, 0xc000810030)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 132 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000565600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2f40, 0xc000810040)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 134 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0002e4310, 0xc000814060)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000822010, 0xc0008240a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 135 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc000812080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 136 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00083a000, 0x1bbe340, 0xc0009329c0, 0xc0005e7c00, 0xc000497a40, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00083a000, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00046b6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005e7ef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00083a000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000822030, 0xc0008240e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 154 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00083a000, 0xc0005a25a0, 0xc000d8a300, 0xc000497a40)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 118 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc000812000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 119 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0007940a0, 0x1bbe340, 0xc000344780, 0xc0007efc00, 0xc0000e35c0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0007940a0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00077fef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0007efef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0007940a0, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000772130, 0xc0007762e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 145 [IO wait]:

internal/poll.runtime_pollWait(0x7f9a4d6f4f00, 0x72, 0x0)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc000565798, 0x72, 0x0, 0x0, 0x194ee55)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Accept(0xc000565780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8

net.(*netFD).accept(0xc000565780, 0xc000561d00, 0xc0001cc000, 0x7f9a4f9616d0)

	/usr/local/go/src/net/fd_unix.go:238 +0x42

net.(*TCPListener).accept(0xc000834480, 0xc000561d30, 0x40c748, 0x30)

	/usr/local/go/src/net/tcpsock_posix.go:139 +0x32

net.(*TCPListener).Accept(0xc000834480, 0x1842ec0, 0xc0006af3e0, 0x1721a80, 0x286f000)

	/usr/local/go/src/net/tcpsock.go:261 +0x47

net/http.(*Server).Serve(0xc0005dc0e0, 0x1be9500, 0xc000834480, 0x0, 0x0)

	/usr/local/go/src/net/http/server.go:2896 +0x286

net/http.(*Server).ListenAndServe(0xc0005dc0e0, 0x4, 0x199a4ef)

	/usr/local/go/src/net/http/server.go:2825 +0xb7

github.com/argoproj/argo/workflow/metrics.runServer.func1(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc0005dc0e0)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:53 +0xf0

created by github.com/argoproj/argo/workflow/metrics.runServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:51 +0x260



goroutine 152 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0007940a0, 0xc0005a25a0, 0xc0005a32c0, 0xc0000e35c0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 108 [select]:

github.com/robfig/cron/v3.(*Cron).run(0xc000287cc0)

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:263 +0x541

created by github.com/robfig/cron/v3.(*Cron).Start

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:222 +0xa8



goroutine 109 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0000ff208, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0000ff1f8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0000ff1e0, 0xc0006ad7e0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000565880)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00081fe00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081fe00, 0x3b9aca00, 0x0, 0xc000838001, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000565880, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012e6c0, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:101 +0x455



goroutine 110 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00080c188, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00080c178)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00080c160, 0xc0008102c0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000812480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00081be00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081be00, 0x3b9aca00, 0x0, 0xc00000fa01, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000812480, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012e870, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:102 +0x4a5



goroutine 111 [semacquire]:

sync.runtime_SemacquireMutex(0xc0000bf60c, 0xc00038ed00, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc0000bf608)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00039b380, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:133 +0xb07

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00039b380)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005fb3d0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005fb3d0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0005fb3d0, 0x3b9aca00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 112 [select]:

golang.org/x/net/http2.(*ClientConn).roundTrip(0xc0002d0c00, 0xc0007f6200, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1096 +0x9a0

golang.org/x/net/http2.(*Transport).RoundTripOpt(0xc00029eba0, 0xc0007f6200, 0xc000910900, 0x754025, 0x0, 0xc0001a10e0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:448 +0x172

golang.org/x/net/http2.(*Transport).RoundTrip(...)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:409

golang.org/x/net/http2.noDialH2RoundTripper.RoundTrip(0xc00029eba0, 0xc0007f6200, 0xc00093e120, 0x5, 0xc0001a1168)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2613 +0x3e

net/http.(*Transport).roundTrip(0xc0001bea00, 0xc0007f6200, 0x16c7600, 0xc00093ce01, 0xc000930420)

	/usr/local/go/src/net/http/transport.go:485 +0xdc1

net/http.(*Transport).RoundTrip(0xc0001bea00, 0xc0007f6200, 0x1954ebc, 0xd, 0xc0009304d8)

	/usr/local/go/src/net/http/roundtrip.go:17 +0x35

k8s.io/client-go/transport.(*bearerAuthRoundTripper).RoundTrip(0xc0003418c0, 0xc0007f6100, 0x1951e42, 0xa, 0xc000930360)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:317 +0x266

k8s.io/client-go/transport.(*userAgentRoundTripper).RoundTrip(0xc000437840, 0xc0007f6000, 0xc000437840, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:167 +0x1c0

net/http.send(0xc0007f6000, 0x1bacd40, 0xc000437840, 0x0, 0x0, 0x0, 0xc0008ee0e0, 0xc0007c2540, 0x1, 0x0)

	/usr/local/go/src/net/http/client.go:250 +0x443

net/http.(*Client).send(0xc000341920, 0xc0007f6000, 0x0, 0x0, 0x0, 0xc0008ee0e0, 0x0, 0x1, 0x3f)

	/usr/local/go/src/net/http/client.go:174 +0xfa

net/http.(*Client).do(0xc000341920, 0xc0007f6000, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/http/client.go:641 +0x3ce

net/http.(*Client).Do(0xc000341920, 0xc0007f6000, 0x0, 0x3, 0xc00093e120)

	/usr/local/go/src/net/http/client.go:509 +0x35

k8s.io/client-go/rest.(*Request).request(0xc0007e0c00, 0xc0009112a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:737 +0x367

k8s.io/client-go/rest.(*Request).Do(0xc0007e0c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:809 +0xc5

github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1.(*workflowTemplates).Get(0xc0004981e0, 0xc0007c22a0, 0x2d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/src/github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1/workflowtemplate.go:57 +0x192

github.com/argoproj/argo/workflow/templateresolution.(*workflowTemplateInterfaceWrapper).Get(0xc000ae02f0, 0xc0007c22a0, 0x2d, 0x1, 0x1, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:30 +0x72

github.com/argoproj/argo/workflow/templateresolution.(*Context).GetTemplateFromRef(0xc000932400, 0xc000bc2ed0, 0x0, 0x1, 0x1)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:129 +0x3e2

github.com/argoproj/argo/workflow/templateresolution.(*Context).GetTemplate(0xc000932400, 0x1bb8900, 0xc0005dd180, 0x0, 0x0, 0x1bb8900)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:154 +0x3d0

github.com/argoproj/argo/workflow/templateresolution.(*Context).resolveTemplateImpl(0xc000932400, 0x1bb8900, 0xc0005dd180, 0x0, 0xe0, 0xd8, 0x18d7f40, 0xc000bc30e0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:212 +0x66d

github.com/argoproj/argo/workflow/templateresolution.(*Context).ResolveTemplate(...)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:180

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc000932100, 0x1bb8900, 0xc0005dd180, 0xc000932400, 0x1bb8e00, 0x28ad5e0, 0xc000911b30, 0x6b5, 0xe, 0xc0007dc380)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:496 +0xe2

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateDAG(0xc000932100, 0xc000bc2a80, 0xc000932400, 0xc00026d680, 0xc00026dbd8, 0xc0000bea00)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:1185 +0x4a5

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc000932100, 0xc00026c240, 0xc000932400, 0x1bb87c0, 0xc000bc27b0, 0xc0007b6bf0, 0x18a4d00, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:427 +0xc40

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc000932100, 0x1bb8a40, 0xc000768630, 0xc0007b6d80, 0x1bb87c0, 0xc000bc27b0, 0xc0007b6bf0, 0x0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169

github.com/argoproj/argo/workflow/validate.ValidateWorkflow(0x1bab600, 0xc000ae02f0, 0x1bab5e0, 0xc000ae0310, 0xc000833000, 0xc0007c2000, 0x0, 0x0, 0x0, 0xc000ae0300, ...)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:234 +0x13c8

github.com/argoproj/argo/workflow/validate.ValidateCronWorkflow(0x1bab600, 0xc000ae02f0, 0x1bab5e0, 0xc000ae0310, 0xc000673000, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:317 +0x1a8

github.com/argoproj/argo/workflow/cron.(*cronWfOperationCtx).validateCronWorkflow(0xc000922230, 0x1bb8b00, 0xc000503d60)

	/go/src/github.com/argoproj/argo/workflow/cron/operator.go:96 +0x178

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00039b380, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:152 +0x438

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00039b380)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005fb3e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005fb3e0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0005fb3e0, 0x3b9aca00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 161 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000049f50, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000049f40)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/util/workqueue.(*Type).Get(0xc0003f5620, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:145 +0xa0

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc00039b380, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:193 +0x57

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc00039b380)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005fb3f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005fb3f0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0005fb3f0, 0x3b9aca00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 162 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000049f50, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000049f40)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/util/workqueue.(*Type).Get(0xc0003f5620, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:145 +0xa0

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc00039b380, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:193 +0x57

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc00039b380)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005fb400)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005fb400, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0005fb400, 0x3b9aca00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 163 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0003f5b00)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 164 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000144540)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 165 [runnable]:

reflect.(*rtype).Elem(0x170ff20, 0xc000470720, 0x15)

	/usr/local/go/src/reflect/type.go:902 +0x149

k8s.io/apimachinery/pkg/runtime.mapFromUnstructured(0x170f560, 0xc000470720, 0x15, 0x170ff20, 0xc000488cb0, 0x195, 0xc00088b538, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:299 +0x274

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000470720, 0x15, 0x170ff20, 0xc000488cb0, 0x195, 0x1708de0, 0xc000f8e550)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:222 +0x502

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0004706f0, 0x15, 0x18f6340, 0xc000488c20, 0x199, 0xc00088b6f8, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0004706f0, 0x15, 0x18f6340, 0xc000488c20, 0x199, 0x1708de0, 0xc000f8e0a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000470690, 0x15, 0x1820100, 0xc000488c00, 0x199, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000470690, 0x15, 0x1820100, 0xc000488c00, 0x199, 0x7f9a4d7006c0, 0x3)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc000470690, 0x1942160, 0xc000488c00, 0xdf949cca322a1b52, 0xc000470720)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc0003aa008, 0xc0009e4030, 0xc000e7d400, 0x1b)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc0000e5180, 0x193f940, 0xc0003aa008)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc0007ac4e0, 0x193f940, 0xc0003aa008)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x288f740, 0x400, 0x7f9a4d6e0ef9)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00088bdd8, 0xc00077ce20, 0x41ebb0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00077cf40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00088bf40, 0xdf8475800, 0x0, 0x1301601, 0xc0000aacc0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000774500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0005fb460)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 166 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000774500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0005fb470)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 140 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc000812480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 141 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc00083a0a0, 0xc000812500, 0xc0005a25a0, 0xc00081cc00, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:193 +0x351

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00083a0a0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:222 +0x17a

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082aef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081cef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00083a0a0, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000822140, 0xc000824360)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 167 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0008ce028, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0008ce018)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0008ce000, 0xc00079c040, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0008d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000752e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000752e00, 0x3b9aca00, 0x0, 0xc0000e6101, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc0008d2000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00067db00, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:62 +0x17d



goroutine 168 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc0005a25a0, 0xc0005fb4e0, 0x1bef540, 0xc000818240)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 169 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc0000aac60, 0x5f5e100, 0x0, 0xc0000aac00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 142 [runnable]:

encoding/json.(*decodeState).valueInterface(0xc0010c4c88, 0x9, 0x9)

	/usr/local/go/src/encoding/json/decode.go:1090 +0x6f

encoding/json.(*decodeState).objectInterface(0xc0010c4c88, 0xc000bc0400)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).object(0xc0010c4c88, 0x1708de0, 0xc000bc0400, 0x194, 0xc0010c4cb0, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:679 +0x2036

encoding/json.(*decodeState).value(0xc0010c4c88, 0x1708de0, 0xc000bc0400, 0x194, 0xc000bc05b0, 0x94)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).object(0xc0010c4c88, 0x166af20, 0xc00000f730, 0x16, 0xc0010c4cb0, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:823 +0x1392

encoding/json.(*decodeState).value(0xc0010c4c88, 0x166af20, 0xc00000f730, 0x16, 0xc000921318, 0x51574e)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).unmarshal(0xc0010c4c88, 0x166af20, 0xc00000f730, 0x0, 0x16b8001)

	/usr/local/go/src/encoding/json/decode.go:179 +0x1fb

encoding/json.(*Decoder).Decode(0xc0010c4c60, 0x166af20, 0xc00000f730, 0x189aa80, 0xc000921301)

	/usr/local/go/src/encoding/json/stream.go:73 +0x166

k8s.io/apimachinery/pkg/util/json.Unmarshal(0xc000d43000, 0x2acf, 0x3000, 0x166af20, 0xc00000f730, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/json/json.go:51 +0x2c1

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToUnstructured(0xc000d43000, 0x2acf, 0x3000, 0xc000921488, 0xc0010f8f98, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:416 +0x8c

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToList(0xc000a2a000, 0x3282f, 0x3fe00, 0xc0007bc000, 0xc000abe280, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:449 +0x31e

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decode(0xc000a2a000, 0x3282f, 0x3fe00, 0x1, 0x1, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:387 +0xcd

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.Decode(0xc000a2a000, 0x3282f, 0x3fe00, 0x0, 0x0, 0x0, 0x409eb5, 0x17566c0, 0x1825800, 0x7f9a4d70ae00, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:337 +0x2e2

k8s.io/apimachinery/pkg/runtime.Decode(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/codec.go:54

k8s.io/client-go/dynamic.(*dynamicResourceClient).List(0xc0008ca050, 0x0, 0x0, 0x0, 0x0, 0xc00086a050, 0x50, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:269 +0x3d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:33 +0x1e9

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000921b98, 0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*ListWatch).List(0xc0007ac460, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:106 +0x16e

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1.2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:184 +0x72

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000921f10, 0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1(0xc000808600, 0xc00083a0a0, 0xc000812500, 0xc000810320, 0xc000810330, 0xc000814300)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:190 +0x1a4

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:175 +0x287



goroutine 149 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0005de1e0, 0x1bbe340, 0xc0010b6080, 0xc0007d1c00, 0xc0007fa4e0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0005de1e0, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082a6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0007d1ef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0005de1e0, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0006ce530, 0xc000834540)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 143 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc00082ddc8, 0x4174be, 0x7f9a4d6b0ee0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0005afdd8, 0xc00082de20, 0xc00082dde8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082df40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005aff40, 0xdf8475800, 0x0, 0x42db01, 0xc000906000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000565500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc000810570)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 62 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0002a2e00, 0xc0007ba120)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0008d6000, 0xc0007840a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 63 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0005a25a0, 0xc0008d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 64 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc0008d8000, 0xc0008d2080, 0xc0005a25a0, 0xc000753c00, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:193 +0x351

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0008d8000, 0xc0005a25a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:222 +0x17a

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007abef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000753ef0, 0x3b9aca00, 0x0, 0x1, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0008d8000, 0xc0005a25a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0008d6020, 0xc0007840e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 177 [runnable]:

encoding/json.stateInString(0xc00025d580, 0x64, 0x0)

	/usr/local/go/src/encoding/json/scanner.go:300 +0x216

encoding/json.(*Decoder).readValue(0xc00025d4a0, 0x0, 0x0, 0x1897600)

	/usr/local/go/src/encoding/json/stream.go:103 +0xe4

encoding/json.(*Decoder).Decode(0xc00025d4a0, 0x166af20, 0xc0004a3d10, 0x189aa80, 0xc0007b3301)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/json.Unmarshal(0xc000e1b000, 0x2ad1, 0x3000, 0x166af20, 0xc0004a3d10, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/json/json.go:51 +0x2c1

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToUnstructured(0xc000e1b000, 0x2ad1, 0x3000, 0xc0007b3488, 0xc001124880, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:416 +0x8c

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToList(0xc000b46000, 0x383fe, 0x3fe00, 0xc00039e640, 0xc00039e600, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:449 +0x31e

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decode(0xc000b46000, 0x383fe, 0x3fe00, 0x1, 0x1, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:387 +0xcd

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.Decode(0xc000b46000, 0x383fe, 0x3fe00, 0x0, 0x0, 0x0, 0x409eb5, 0x17566c0, 0x1825800, 0x7f9a4d70ae00, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:337 +0x2e2

k8s.io/apimachinery/pkg/runtime.Decode(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/codec.go:54

k8s.io/client-go/dynamic.(*dynamicResourceClient).List(0xc0008b40a0, 0x0, 0x0, 0x0, 0x0, 0xc000916150, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:269 +0x3d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:33 +0x1e9

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc0007b3b98, 0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*ListWatch).List(0xc0006a95e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:106 +0x16e

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1.2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:184 +0x72

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc0007b3f10, 0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1(0xc0007c0180, 0xc0008d8000, 0xc0008d2080, 0xc00079c0a0, 0xc00079c0b0, 0xc0007ba240)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:190 +0x1a4

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:175 +0x287



goroutine 170 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00081ddd8, 0xc00082d620, 0x41ebb0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00082d740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081df40, 0xdf8475800, 0x0, 0xc0000aab01, 0xc0008144e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000774480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001444c0, 0xc0005fb4f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 171 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000774480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001444c0, 0xc0005fb500)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 40 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0005de1e0, 0xc0005a25a0, 0xc000db2f00, 0xc0007fa4e0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 83 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xbfc03149482e8923, 0x33813b2)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0005aedd8, 0xc000466620, 0x41ebb0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000466740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005aef40, 0xdf8475800, 0x0, 0x1, 0xc000122540)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000774300)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001443e0, 0xc0003da190)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 84 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000774300)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001443e0, 0xc0003da1e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 144 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000565500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc000810580)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 194 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000565580)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0008105a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 195 [runnable]:

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x193c3e0, 0x1e, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:269 +0x42a

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0005958f0, 0x15, 0x193c3e0, 0xc0006a4400, 0x199, 0xc00041bce8, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000905780, 0x194, 0x193c3e0, 0xc0006a4400, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc000b17c40, 0x97, 0x167a900, 0xc000766518, 0x197, 0xc00088d538, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc000b17c40, 0x97, 0x167a900, 0xc000766518, 0x197, 0x1708de0, 0xc000bae060)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000595860, 0x15, 0x1923380, 0xc000766518, 0x199, 0xc00088d6f8, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000595860, 0x15, 0x1923380, 0xc000766518, 0x199, 0x1708de0, 0xc000b88470)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0005956e0, 0x15, 0x1820100, 0xc000766400, 0x199, 0x7f9a4d94ffff, 0x400)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0005956e0, 0x15, 0x1820100, 0xc000766400, 0x199, 0x7f9a4d4b1b50, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc0005956e0, 0x1942160, 0xc000766400, 0x6389c7cba51032f, 0xc000595800)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc0003aa000, 0xc000ab6000, 0xc000e7cf00, 0x1f)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc0000e5180, 0x193f940, 0xc0003aa000)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc0007ac4e0, 0x193f940, 0xc0003aa000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xc0007a4d90, 0x430340)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00088ddd8, 0xc0007a4e20, 0x41eb4d)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0007a4f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00088df40, 0xdf8475800, 0x0, 0x1, 0xc0007ba2a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000774500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0008105b0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 196 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000774500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0002a2e60, 0xc0008105c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 202 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0009317a0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000931790)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc000931788, 0xc00040e600, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc000931760, 0xc00040e600, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc000931ce0, 0xc000581560, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc000931ce0, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc000931ce0, 0x1742800, 0xc000581560, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000748510, 0xc0000b8000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0008b43c0, 0x0, 0x1bbde40, 0xc00090a8c0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000581540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00090a880)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 38 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0007c7bc0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0007c7bb0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc0007c7ba8, 0xc00084a000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc0007c7b80, 0xc00084a000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc000900420, 0xc000498340, 0x142a69c)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc000900420, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc000900420, 0x1742800, 0xc000498340, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0006d6360, 0xc000846000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0005521e0, 0x0, 0x1bbde40, 0xc000344880, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000498320, 0x0, 0x18, 0xc00082b670, 0x12c8b60, 0x1a07710, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000344780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 179 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000900e00, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000900df0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc000900de8, 0xc000940200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc000900dc0, 0xc000940200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc000930840, 0xc000abe060, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc000930840, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc000930840, 0x1742800, 0xc000abe060, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000bc34a0, 0xc000846800, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000586230, 0x0, 0x1bbde40, 0xc000932ac0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000abe040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0009329c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 71 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0007c77a0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0007c7790)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc0007c7788, 0xc0010dc000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc0007c7760, 0xc0010dc000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc0010c4160, 0xc00000c0e0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc0010c4160, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc0010c4160, 0x1742800, 0xc00000c0e0, 0x421305, 0xc0010b60c0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000cc6120, 0xc000106400, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0007c4050, 0x0, 0x1bbde40, 0xc0010b60c0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00000c0c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0010b6080)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe













time="2020-07-28T16:10:27Z" level=info msg="config map" name=workflow-controller-configmap

time="2020-07-28T16:10:27Z" level=info msg="Configuration:\nartifactRepository: {}\nmetricsConfig: {}\nnodeEvents: {}\npodSpecLogStrategy: {}\nsso:\n  clientId:\n    key: \"\"\n  clientSecret:\n    key: \"\"\n  issuer: \"\"\n  redirectUrl: \"\"\ntelemetryConfig: {}\n"

time="2020-07-28T16:10:27Z" level=info msg="Persistence configuration disabled"

time="2020-07-28T16:10:27Z" level=info msg="Starting Workflow Controller" version=v2.9.1

time="2020-07-28T16:10:27Z" level=info msg="Workers: workflow: 32, pod: 32"

time="2020-07-28T16:10:27Z" level=info msg="Performing periodic GC every 5m0s"

time="2020-07-28T16:10:27Z" level=info msg="Starting prometheus metrics server at localhost:9090/metrics"

time="2020-07-28T16:10:27Z" level=info msg="Starting CronWorkflow controller"

time="2020-07-28T16:10:27Z" level=info msg="Persistence disabled - so archived workflow GC disabled - you must restart the controller if you enable this"

time="2020-07-28T16:10:27Z" level=info msg="Starting workflow TTL controller (resync 20m0s)"

time="2020-07-28T16:10:27Z" level=info msg="Processing release/ideal-rankings-workflow"

time="2020-07-28T16:10:27Z" level=info msg="Processing release/seo-impressions-workflow"

fatal error: concurrent map writes



goroutine 172 [running]:

runtime.throw(0x1960bd9, 0x15)

	/usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc00083dbd0 sp=0xc00083dba0 pc=0x42e5d2

runtime.mapassign_faststr(0x170e600, 0xc000602d50, 0xc000dda000, 0x19, 0x28aea20)

	/usr/local/go/src/runtime/map_faststr.go:211 +0x417 fp=0xc00083dc38 sp=0xc00083dbd0 pc=0x413127

github.com/argoproj/argo/workflow/metrics.(*Metrics).WorkflowAdded(0xc0004e4600, 0xc000dda000, 0x19, 0xc000eaba85, 0x9)

	/go/src/github.com/argoproj/argo/workflow/metrics/metrics.go:95 +0x99 fp=0xc00083dc78 sp=0xc00083dc38 pc=0x1495f99

github.com/argoproj/argo/workflow/controller.(*WorkflowController).addWorkflowInformerHandlers.func5(0x193f940, 0xc000172650)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:592 +0xbd fp=0xc00083dcd0 sp=0xc00083dc78 pc=0x15b4e6d

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd(0xc000608ca0, 0x193f940, 0xc000172650)

	<autogenerated>:1 +0x5a fp=0xc00083dcf0 sp=0xc00083dcd0 pc=0x130b5da

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218 fp=0xc00083dd68 sp=0xc00083dcf0 pc=0x13093d8

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00083ddd8, 0x0, 0xc000749038)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51 fp=0xc00083dd90 sp=0xc00083dd68 pc=0x12c84b1

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79 fp=0xc00083ddf8 sp=0xc00083dd90 pc=0x1309499

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000541740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e fp=0xc00083de68 sp=0xc00083ddf8 pc=0x12c8b6e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00083df40, 0xdf8475800, 0x0, 0xc0000aaa01, 0xc000744600)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8 fp=0xc00083df18 sp=0xc00083de68 pc=0x12c8108

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00062e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b fp=0xc00083df68 sp=0xc00083df18 pc=0x13035db

k8s.io/client-go/tools/cache.(*processorListener).run-fm()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:593 +0x2a fp=0xc00083df80 sp=0xc00083df68 pc=0x130b2da

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d66f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59 fp=0xc00083dfd0 sp=0xc00083df80 pc=0x12c8ae9

runtime.goexit()

	/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00083dfd8 sp=0xc00083dfd0 pc=0x45b741

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 1 [select (no cases)]:

main.NewRootCommand.func1(0xc00011cc80, 0xc00028ff00, 0x0, 0x4, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:89 +0x3f4

github.com/spf13/cobra.(*Command).execute(0xc00011cc80, 0xc00003a060, 0x4, 0x4, 0xc00011cc80, 0xc00003a060)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:762 +0x460

github.com/spf13/cobra.(*Command).ExecuteC(0xc00011cc80, 0xc00006a750, 0xc0000a5f50, 0x40576f)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:852 +0x2ea

github.com/spf13/cobra.(*Command).Execute(...)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:800

main.main()

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:111 +0x2b



goroutine 6 [chan receive]:

k8s.io/klog.(*loggingT).flushDaemon(0x288fb80)

	/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b

created by k8s.io/klog.init.0

	/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd6



goroutine 7 [syscall]:

os/signal.signal_recv(0x0)

	/usr/local/go/src/runtime/sigqueue.go:147 +0x9c

os/signal.loop()

	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22

created by os/signal.init.0

	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41



goroutine 38 [chan receive]:

github.com/argoproj/pkg/stats.RegisterStackDumper.func1()

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:31 +0xa0

created by github.com/argoproj/pkg/stats.RegisterStackDumper

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:27 +0x35



goroutine 40 [chan receive]:

github.com/argoproj/pkg/stats.StartStatsTicker.func1(0xc00003a550)

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:19 +0x37

created by github.com/argoproj/pkg/stats.StartStatsTicker

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:17 +0x4d



goroutine 41 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000186540)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 42 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000119f80)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 43 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000186660)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 44 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000130000)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 130 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc0001563f0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 131 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00008d740)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 16 [IO wait]:

internal/poll.runtime_pollWait(0x7fbdc4099fd0, 0x72, 0xffffffffffffffff)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc00038e198, 0x72, 0x10200, 0x102de, 0xffffffffffffffff)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Read(0xc00038e180, 0xc000a7a000, 0x102de, 0x102de, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf

net.(*netFD).Read(0xc00038e180, 0xc000a7a000, 0x102de, 0x102de, 0x203000, 0x0, 0x10287)

	/usr/local/go/src/net/fd_unix.go:202 +0x4f

net.(*conn).Read(0xc0002fa000, 0xc000a7a000, 0x102de, 0x102de, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/net.go:184 +0x68

crypto/tls.(*atLeastReader).Read(0xc000974060, 0xc000a7a000, 0x102de, 0x102de, 0x1000, 0x400, 0xc0000b3970)

	/usr/local/go/src/crypto/tls/conn.go:780 +0x60

bytes.(*Buffer).ReadFrom(0xc000097ad8, 0x1bab100, 0xc000974060, 0x409eb5, 0x1755ec0, 0x18e2f80)

	/usr/local/go/src/bytes/buffer.go:204 +0xb4

crypto/tls.(*Conn).readFromUntil(0xc000097880, 0x1bacec0, 0xc0002fa000, 0x5, 0xc0002fa000, 0x203000)

	/usr/local/go/src/crypto/tls/conn.go:802 +0xec

crypto/tls.(*Conn).readRecordOrCCS(0xc000097880, 0x0, 0x0, 0xc000974000)

	/usr/local/go/src/crypto/tls/conn.go:609 +0x124

crypto/tls.(*Conn).readRecord(...)

	/usr/local/go/src/crypto/tls/conn.go:577

crypto/tls.(*Conn).Read(0xc000097880, 0xc000507000, 0x1000, 0x1000, 0x0, 0x0, 0x0)

	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161

bufio.(*Reader).Read(0xc00008d500, 0xc000514038, 0x9, 0x9, 0xc0000b3d10, 0x0, 0x8a2bb5)

	/usr/local/go/src/bufio/bufio.go:226 +0x26a

io.ReadAtLeast(0x1baaee0, 0xc00008d500, 0xc000514038, 0x9, 0x9, 0x9, 0xc000080110, 0x0, 0x1bab3c0)

	/usr/local/go/src/io/io.go:310 +0x87

io.ReadFull(...)

	/usr/local/go/src/io/io.go:329

golang.org/x/net/http2.readFrameHeader(0xc000514038, 0x9, 0x9, 0x1baaee0, 0xc00008d500, 0x0, 0x0, 0xc000868120, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x87

golang.org/x/net/http2.(*Framer).ReadFrame(0xc000514000, 0xc000868120, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0xa1

golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000b3fb8, 0xc000069fa8, 0x88bb98)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1746 +0x8c

golang.org/x/net/http2.(*ClientConn).readLoop(0xc000480900)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1674 +0xa3

created by golang.org/x/net/http2.(*Transport).newClientConn

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:674 +0x62f



goroutine 129 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00008d620)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 47 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc0004ef8c0, 0xc000657d18, 0xc00001a7e0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc000657d18, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc00001a780, 0xc000657e78, 0x1, 0x1, 0xc000189640)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run(0xc00048ae00, 0x1bef540, 0xc000189640, 0x20, 0x20)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:163 +0x8f3

created by main.NewRootCommand.func1

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:86 +0x3ef



goroutine 48 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0000eef48, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0000eef38)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0000eef20, 0xc00049e270, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000552080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000314ec8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000314ec8, 0x3b9aca00, 0x0, 0xc000570101, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000552080, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

github.com/argoproj/argo/config.(*controller).Run(0xc0003b1830, 0xc00001a780, 0xc000630510)

	/go/src/github.com/argoproj/argo/config/controller.go:119 +0x384

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:148 +0x44c



goroutine 113 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0000eeff8, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0000eefe8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0000eefd0, 0xc000569950, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000552600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000424e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000424e00, 0x3b9aca00, 0x0, 0xc0005fa601, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000552600, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0006141b0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:149 +0x4a2



goroutine 114 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0000ef158, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0000ef148)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0000ef130, 0xc000569c30, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000552980)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000426e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000426e00, 0x3b9aca00, 0x0, 0xc000014f01, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000552980, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0006142d0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:150 +0x52c



goroutine 115 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000166918, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000166908)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001668f0, 0xc0006d6740, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00040f400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000836e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000836e00, 0x3b9aca00, 0x0, 0xc000579001, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00040f400, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000614240, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:151 +0x582



goroutine 116 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podLabeler(0xc00048ae00, 0xc00001a780)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:215 +0xda

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:152 +0x5cb



goroutine 117 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podGarbageCollector(0xc00048ae00, 0xc00001a780)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:241 +0xd4

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:153 +0x614



goroutine 118 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).workflowGarbageCollector(0xc00048ae00, 0xc00001a780)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:275 +0x18e

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:154 +0x65d



goroutine 50 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0000d02a0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 120 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc00073c080, 0xc000838e40, 0xc000744120, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc000651e40, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc00001a780, 0xc000651f28, 0x1, 0x1, 0xc00072cf50)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run(0xc00026a0f0, 0xc00001a780, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:63 +0x211

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runTTLController(0xc00048ae00, 0x1bef540, 0xc000189640)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:123 +0x86

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:157 +0x6e5



goroutine 121 [chan receive]:

github.com/argoproj/argo/workflow/cron.(*Controller).Run(0xc00014cfc0, 0x1bef540, 0xc000189640)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:112 +0x66a

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runCronController(0xc00048ae00, 0x1bef540, 0xc000189640)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:131 +0xc0

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:158 +0x724



goroutine 123 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc00001a780, 0xc000630540, 0x1bef540, 0xc000189980)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 124 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc00001a8a0, 0x5f5e100, 0x0, 0xc00001a840)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 20 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc000552080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 21 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00056e140, 0x1bbe340, 0xc00087b340, 0xc000655c00, 0xc000187c20, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00056e140, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000546ef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000655ef0, 0x3b9aca00, 0x0, 0x192ee01, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00056e140, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00053c070, 0xc000420380)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 127 [chan receive]:

github.com/argoproj/argo/workflow/metrics.runServer(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc0000d7130, 0x1bef540, 0xc000189640)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:59 +0x291

created by github.com/argoproj/argo/workflow/metrics.(*Metrics).RunServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:36 +0x28f



goroutine 147 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00056e140, 0xc00001a780, 0xc00001a900, 0xc000187c20)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 145 [IO wait]:

internal/poll.runtime_pollWait(0x7fbdc4099f00, 0x72, 0x0)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc00062e418, 0x72, 0x0, 0x0, 0x194ee55)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Accept(0xc00062e400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8

net.(*netFD).accept(0xc00062e400, 0xc000653d00, 0xc00047e000, 0x7fbdc63066d0)

	/usr/local/go/src/net/fd_unix.go:238 +0x42

net.(*TCPListener).accept(0xc000609aa0, 0xc000653d30, 0x40c748, 0x30)

	/usr/local/go/src/net/tcpsock_posix.go:139 +0x32

net.(*TCPListener).Accept(0xc000609aa0, 0x1842ec0, 0xc000603770, 0x1721a80, 0x286f000)

	/usr/local/go/src/net/tcpsock.go:261 +0x47

net/http.(*Server).Serve(0xc0006a6000, 0x1be9500, 0xc000609aa0, 0x0, 0x0)

	/usr/local/go/src/net/http/server.go:2896 +0x286

net/http.(*Server).ListenAndServe(0xc0006a6000, 0x4, 0x199a4ef)

	/usr/local/go/src/net/http/server.go:2825 +0xb7

github.com/argoproj/argo/workflow/metrics.runServer.func1(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc0006a6000)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:53 +0xf0

created by github.com/argoproj/argo/workflow/metrics.runServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:51 +0x260



goroutine 132 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc0001564d0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 133 [select]:

github.com/robfig/cron/v3.(*Cron).run(0xc000516140)

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:263 +0x541

created by github.com/robfig/cron/v3.(*Cron).Start

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:222 +0xa8



goroutine 134 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000166708, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0001666f8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001666e0, 0xc0006d6120, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00040e600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000728e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000728e00, 0x3b9aca00, 0x0, 0xc0003c1501, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00040e600, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000012120, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:101 +0x455



goroutine 135 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000166868, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000166858)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000166840, 0xc0006d63e0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00040ee00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00072ae00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00072ae00, 0x3b9aca00, 0x0, 0xc0002fa001, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00040ee00, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0000122d0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:102 +0x4a5



goroutine 136 [semacquire]:

sync.runtime_SemacquireMutex(0xc00035aa7c, 0xc0006a2300, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc00035aa78)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00014cfc0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:133 +0xb07

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00014cfc0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d60d0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d60d0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006d60d0, 0x3b9aca00, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 137 [runnable]:

github.com/modern-go/reflect2.(*UnsafeStructField).Type(0xc0009aee00, 0xc000671ce0, 0x54)

	/go/pkg/mod/github.com/modern-go/[email protected]/unsafe_field.go:36 +0x68

github.com/json-iterator/go.describeStruct(0xc000795710, 0x1c1e7c0, 0xc00044e4b0, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa4b

github.com/json-iterator/go.describeStruct(0xc000795710, 0x1c1e7c0, 0xc00086a480, 0x1818840)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:350 +0xb0a

github.com/json-iterator/go.encoderOfStruct(0xc000795710, 0x1c1e7c0, 0xc00086a480, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_encoder.go:18 +0x5a

github.com/json-iterator/go._createEncoderOfType(0xc000795710, 0x1c1e7c0, 0xc00086a480, 0xc000d595f8, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:275 +0x441

github.com/json-iterator/go.createEncoderOfType(0xc000795710, 0x1c1e7c0, 0xc00086a480, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:245 +0x10b

github.com/json-iterator/go.encoderOfType(0xc000795710, 0x1c1e7c0, 0xc00086a480, 0x14, 0x94878f)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:227 +0x78

github.com/json-iterator/go.encoderOfOptional(0xc000795710, 0x1c1e680, 0xc00086a420, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_optional.go:22 +0x54

github.com/json-iterator/go._createEncoderOfType(0xc000795710, 0x1c1e680, 0xc00086a420, 0xc000d5c498, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:283 +0x4df

github.com/json-iterator/go.createEncoderOfType(0xc000795710, 0x1c1e680, 0xc00086a420, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:245 +0x10b

github.com/json-iterator/go.encoderOfType(0xc000795710, 0x1c1e680, 0xc00086a420, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:227 +0x78

github.com/json-iterator/go.describeStruct(0xc000c46660, 0x1c1e7c0, 0xc00086a330, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:384 +0x8e7

github.com/json-iterator/go.decoderOfStruct(0xc000c46660, 0x1c1e7c0, 0xc00086a330, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc000c46660, 0x1c1e7c0, 0xc00086a330, 0xc000d55c58, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc000c46660, 0x1c1e7c0, 0xc00086a330, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000c46660, 0x1c1e7c0, 0xc00086a330, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.decoderOfSlice(0xc000795440, 0x1c1e720, 0xc00073b380, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_slice.go:12 +0x105

github.com/json-iterator/go._createDecoderOfType(0xc000795440, 0x1c1e720, 0xc00073b380, 0xc000d554d8, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:180 +0x4ff

github.com/json-iterator/go.createDecoderOfType(0xc000795440, 0x1c1e720, 0xc00073b380, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000795440, 0x1c1e720, 0xc00073b380, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.describeStruct(0xc000795410, 0x1c1e7c0, 0xc00086a1b0, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa5c

github.com/json-iterator/go.decoderOfStruct(0xc000795410, 0x1c1e7c0, 0xc00086a1b0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc000795410, 0x1c1e7c0, 0xc00086a1b0, 0xc000d552d8, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc000795410, 0x1c1e7c0, 0xc00086a1b0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000795410, 0x1c1e7c0, 0xc00086a1b0, 0xc000c46d98, 0x93e167)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.decoderOfOptional(0xc000795410, 0x1c1e680, 0xc00086a150, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_optional.go:12 +0x68

github.com/json-iterator/go._createDecoderOfType(0xc000795410, 0x1c1e680, 0xc00086a150, 0xc000d56828, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:184 +0x554

github.com/json-iterator/go.createDecoderOfType(0xc000795410, 0x1c1e680, 0xc00086a150, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000795410, 0x1c1e680, 0xc00086a150, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.describeStruct(0xc000c476c8, 0x1c1e7c0, 0xc0003b0b10, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa5c

github.com/json-iterator/go.describeStruct(0xc000c476c8, 0x1c1e7c0, 0xc0003b0690, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:350 +0xb0a

github.com/json-iterator/go.decoderOfStruct(0xc000c476c8, 0x1c1e7c0, 0xc0003b0690, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc000c476c8, 0x1c1e7c0, 0xc0003b0690, 0xc00005eda8, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc000c476c8, 0x1c1e7c0, 0xc0003b0690, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000c476c8, 0x1c1e7c0, 0xc0003b0690, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.decoderOfSlice(0xc000969ef0, 0x1c1e720, 0xc000189380, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_slice.go:12 +0x105

github.com/json-iterator/go._createDecoderOfType(0xc000969ef0, 0x1c1e720, 0xc000189380, 0xc00005d338, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:180 +0x4ff

github.com/json-iterator/go.createDecoderOfType(0xc000969ef0, 0x1c1e720, 0xc000189380, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000969ef0, 0x1c1e720, 0xc000189380, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.describeStruct(0xc000c47e10, 0x1c1e7c0, 0xc00061b2c0, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa5c

github.com/json-iterator/go.decoderOfStruct(0xc000c47e10, 0x1c1e7c0, 0xc00061b2c0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc000c47e10, 0x1c1e7c0, 0xc00061b2c0, 0xc001129998, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc000c47e10, 0x1c1e7c0, 0xc00061b2c0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000c47e10, 0x1c1e7c0, 0xc00061b2c0, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.decoderOfSlice(0xc0004b0a80, 0x1c1e720, 0xc0003c8600, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_slice.go:12 +0x105

github.com/json-iterator/go._createDecoderOfType(0xc0004b0a80, 0x1c1e720, 0xc0003c8600, 0xc001129a68, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:180 +0x4ff

github.com/json-iterator/go.createDecoderOfType(0xc0004b0a80, 0x1c1e720, 0xc0003c8600, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc0004b0a80, 0x1c1e720, 0xc0003c8600, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.describeStruct(0xc0004b0a20, 0x1c1e7c0, 0xc00061b1a0, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa5c

github.com/json-iterator/go.describeStruct(0xc0004b0a20, 0x1c1e7c0, 0xc00061b0e0, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:350 +0xb0a

github.com/json-iterator/go.decoderOfStruct(0xc0004b0a20, 0x1c1e7c0, 0xc00061b0e0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc0004b0a20, 0x1c1e7c0, 0xc00061b0e0, 0xc001129a58, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc0004b0a20, 0x1c1e7c0, 0xc00061b0e0, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc0004b0a20, 0x1c1e7c0, 0xc00061b0e0, 0x194bc08, 0x1)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.describeStruct(0xc000c48cd0, 0x1c1e7c0, 0xc000500a20, 0x25e90)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_extension.go:380 +0xa5c

github.com/json-iterator/go.decoderOfStruct(0xc000c48cd0, 0x1c1e7c0, 0xc000500a20, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect_struct_decoder.go:14 +0xb6

github.com/json-iterator/go._createDecoderOfType(0xc000c48cd0, 0x1c1e7c0, 0xc000500a20, 0xc00079a088, 0x1853a20)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:176 +0x4aa

github.com/json-iterator/go.createDecoderOfType(0xc000c48cd0, 0x1c1e7c0, 0xc000500a20, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:142 +0x10b

github.com/json-iterator/go.decoderOfType(0xc000c48cd0, 0x1c1e7c0, 0xc000500a20, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:124 +0x78

github.com/json-iterator/go.(*frozenConfig).DecoderOf(0xc000122aa0, 0x1c1e680, 0xc000500a80, 0x1c1e680, 0xc000500a80)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:114 +0x111

github.com/json-iterator/go.(*Iterator).ReadVal(0xc00059a5a0, 0x193e200, 0xc000014600)

	/go/pkg/mod/github.com/json-iterator/[email protected]/reflect.go:72 +0x1ce

github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000122aa0, 0xc00076f000, 0xa30, 0xe00, 0x193e200, 0xc000014600, 0x0, 0x0)

	/go/pkg/mod/github.com/json-iterator/[email protected]/config.go:348 +0xd2

k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc000049d00, 0xc00076f000, 0xa30, 0xe00, 0x0, 0x1bb8a80, 0xc000014600, 0x1, 0x0, 0x1bacf60, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/json/json.go:275 +0x7d5

k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1bac7e0, 0xc000049d00, 0xc00076f000, 0xa30, 0xe00, 0x0, 0x1bb8a80, 0xc000014600, 0xc00049f140, 0xc000dbc400, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/helper.go:252 +0x97

k8s.io/client-go/rest.Result.Into(0xc00076f000, 0xa30, 0xe00, 0xc00053c020, 0x10, 0x0, 0x0, 0xc8, 0x1bae460, 0xc00034b8e0, ...)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:1112 +0xba

github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1.(*workflowTemplates).Get(0xc000550100, 0xc000afc2a0, 0x18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/src/github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1/workflowtemplate.go:58 +0x1eb

github.com/argoproj/argo/workflow/templateresolution.(*workflowTemplateInterfaceWrapper).Get(0xc0004a0800, 0xc000afc2a0, 0x18, 0x1, 0x1, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:30 +0x72

github.com/argoproj/argo/workflow/templateresolution.(*Context).GetTemplateFromRef(0xc000db2440, 0xc00049ec00, 0x0, 0x1, 0x1)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:129 +0x3e2

github.com/argoproj/argo/workflow/templateresolution.(*Context).GetTemplate(0xc000db2440, 0x1bb8900, 0xc0007ba9a0, 0x0, 0x0, 0x1bb8900)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:154 +0x3d0

github.com/argoproj/argo/workflow/templateresolution.(*Context).resolveTemplateImpl(0xc000db2440, 0x1bb8900, 0xc0007ba9a0, 0x0, 0xe0, 0xd8, 0x18d7f40, 0xc00049ef90, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:212 +0x66d

github.com/argoproj/argo/workflow/templateresolution.(*Context).ResolveTemplate(...)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:180

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc000db2340, 0x1bb8900, 0xc0007ba9a0, 0xc000db2440, 0x1bb8e00, 0x28ad5e0, 0xc000c49b30, 0x437, 0xe, 0xc0003c0e00)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:496 +0xe2

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateDAG(0xc000db2340, 0xc00049e7b0, 0xc000db2440, 0xc0003e3200, 0xc0003e3528, 0xc000b0a300)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:1185 +0x4a5

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc000db2340, 0xc0003d5200, 0xc000db2440, 0x1bb87c0, 0xc00049e510, 0xc000bb2bf0, 0x18a4d00, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:427 +0xc40

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc000db2340, 0x1bb8a40, 0xc000b24210, 0xc000bb2d80, 0x1bb87c0, 0xc00049e510, 0xc000bb2bf0, 0x0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169

github.com/argoproj/argo/workflow/validate.ValidateWorkflow(0x1bab600, 0xc0004a0800, 0x1bab5e0, 0xc0004a08a0, 0xc000578c00, 0xc000b0c000, 0x0, 0x0, 0x0, 0xc0004a0850, ...)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:234 +0x13c8

github.com/argoproj/argo/workflow/validate.ValidateCronWorkflow(0x1bab600, 0xc0004a0800, 0x1bab5e0, 0xc0004a08a0, 0xc0005f1000, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:317 +0x1a8

github.com/argoproj/argo/workflow/cron.(*cronWfOperationCtx).validateCronWorkflow(0xc000130a80, 0x1bb8b00, 0xc00034b770)

	/go/src/github.com/argoproj/argo/workflow/cron/operator.go:96 +0x178

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc00014cfc0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:152 +0x438

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc00014cfc0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d60e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d60e0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006d60e0, 0x3b9aca00, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 138 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0003c8210, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0003c8200)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/util/workqueue.(*Type).Get(0xc00008d620, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:145 +0xa0

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc00014cfc0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:193 +0x57

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc00014cfc0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d60f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d60f0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006d60f0, 0x3b9aca00, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 139 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0003c8210, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0003c8200)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/util/workqueue.(*Type).Get(0xc00008d620, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:145 +0xa0

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc00014cfc0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:193 +0x57

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc00014cfc0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d6100)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d6100, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006d6100, 0x3b9aca00, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 178 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00081add8, 0x0, 0xc000546df8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000546f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00081af40, 0xdf8475800, 0x0, 0x42db01, 0xc0001b0840)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00062e280)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c990, 0xc000710460)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 24 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00028c850, 0xc0001b02a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00053c1d0, 0xc0004207c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 25 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc000552600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 26 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc00056e1e0, 0xc000552680, 0xc00001a780, 0xc000425c00, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:193 +0x351

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00056e1e0, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:222 +0x17a

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005476f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000425ef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00056e1e0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00053c1f0, 0xc000420840)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 27 [runnable]:

encoding/json.stateBeginString(0xc001199c60, 0x1897622, 0x1)

	/usr/local/go/src/encoding/json/scanner.go:229 +0x1ff

encoding/json.stateBeginStringOrEmpty(0xc001199c60, 0x22, 0x2)

	/usr/local/go/src/encoding/json/scanner.go:225 +0x95

encoding/json.(*Decoder).readValue(0xc001199b80, 0x0, 0x0, 0x1897600)

	/usr/local/go/src/encoding/json/stream.go:103 +0xe4

encoding/json.(*Decoder).Decode(0xc001199b80, 0x166af20, 0xc00053a508, 0x189aa80, 0xc0006e3301)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/json.Unmarshal(0xc000d78000, 0x2ace, 0x3000, 0x166af20, 0xc00053a508, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/json/json.go:51 +0x2c1

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToUnstructured(0xc000d78000, 0x2ace, 0x3000, 0xc0006e3488, 0xc000cbfc58, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:416 +0x8c

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToList(0xc000b2c000, 0x383fe, 0x3fe00, 0xc000420740, 0xc000420700, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:449 +0x31e

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decode(0xc000b2c000, 0x383fe, 0x3fe00, 0x1, 0x1, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:387 +0xcd

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.Decode(0xc000b2c000, 0x383fe, 0x3fe00, 0x0, 0x0, 0x0, 0x409eb5, 0x17566c0, 0x1825800, 0x7fbdc3eefd00, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:337 +0x2e2

k8s.io/apimachinery/pkg/runtime.Decode(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/codec.go:54

k8s.io/client-go/dynamic.(*dynamicResourceClient).List(0xc000254190, 0x0, 0x0, 0x0, 0x0, 0xc0000405d0, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:269 +0x3d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:33 +0x1e9

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc000046028, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc0006e3b98, 0x1bef580, 0xc000046028, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*ListWatch).List(0xc000608c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:106 +0x16e

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1.2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:184 +0x72

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc000046020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc0006e3f10, 0x1bef580, 0xc000046020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1(0xc0001d26c0, 0xc00056e1e0, 0xc000552680, 0xc0005699b0, 0xc0005699c0, 0xc0001b0360)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:190 +0x1a4

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:175 +0x287



goroutine 166 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc00008dd40, 0xc0005427c8, 0x1435581)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000827dd8, 0xaf184e7, 0xed6b244f3)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000542740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000827f40, 0xdf8475800, 0x0, 0x42db01, 0xc0000aaf60)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00040e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001566f0, 0xc0006d6690)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 141 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000156690, 0xc0000aaba0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035aae0, 0xc00033eee0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 142 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc00040e600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 51 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc0002ce230)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 143 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0005161e0, 0x1bbe340, 0xc000db26c0, 0xc000a6bc00, 0xc0001d3080, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0005161e0, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d36f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000a6bef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0005161e0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035ab20, 0xc00033ef20)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 153 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0005161e0, 0xc00001a780, 0xc000a134a0, 0xc0001d3080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 52 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).add(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:560

k8s.io/client-go/tools/cache.(*sharedProcessor).distribute(0xc00028c850, 0x177f0a0, 0xc000de0010, 0xc000de0001)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:459 +0xe7

k8s.io/client-go/tools/cache.(*sharedIndexInformer).HandleDeltas(0xc0006141b0, 0x1757640, 0xc000ddc060, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:416 +0x2ab

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0007e2000, 0xc000710130, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:422 +0x230

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0007d2080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00072de00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00072de00, 0x3b9aca00, 0x0, 0xc0005fa601, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc0007d2080, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0006141b0, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:62 +0x17d



goroutine 53 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc00001a780, 0xc000710110, 0x1bef540, 0xc00073a0c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 54 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc0007441e0, 0x5f5e100, 0x0, 0xc000744180)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 60 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xbfc0315ccaf30b62, 0x43106f6)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00082add8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000733f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00082af40, 0xdf8475800, 0x0, 0x42db01, 0xc000744480)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00062e180)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710400)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 56 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00028c850, 0xc0007442a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000740030, 0xc000388280)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 57 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc0007d2080)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 58 [runnable]:

net/url.escape(0xc001149b00, 0x2c, 0x6, 0x0, 0xd)

	/usr/local/go/src/net/url/url.go:335 +0x392

net/url.QueryEscape(...)

	/usr/local/go/src/net/url/url.go:273

net/url.Values.Encode(0xc000485280, 0x0, 0x0)

	/usr/local/go/src/net/url/url.go:930 +0x507

k8s.io/client-go/rest.(*Request).URL(0xc000842300, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:452 +0x427

k8s.io/client-go/rest.(*Request).WatchWithSpecificDecoders(0xc000842300, 0xc0004857c8, 0x7fbdc3efca08, 0x28ad300, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:559 +0x72

k8s.io/client-go/dynamic.(*dynamicResourceClient).Watch(0xc00003a320, 0x0, 0x0, 0x0, 0x0, 0xc001149950, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:309 +0x8d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000eaa0a8, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:39 +0x1e9

k8s.io/client-go/tools/cache.(*ListWatch).Watch(0xc000608c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:113 +0x6c

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0007e4000, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:275 +0x420

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000541ef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000485ef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0007e4000, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000740050, 0xc0003882e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 201 [runnable]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0007e4000, 0xc00001a780, 0xc000745da0, 0xc0001d2540)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 179 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00062e280)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c990, 0xc000710470)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 29 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00028c9a0, 0xc0001b0480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00053c370, 0xc000420dc0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 30 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc000552980)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 168 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000828dd8, 0x0, 0xc0006d35f8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d3740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000828f40, 0xdf8475800, 0x0, 0x42db01, 0xc0000aafc0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00040e500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001567d0, 0xc0006d66b0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 162 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000156770, 0xc0000aacc0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035ac10, 0xc00033f220)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 163 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc00040ee00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 31 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00056e280, 0x1bbe340, 0xc000a32080, 0xc000a71c00, 0xc0007d5260, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00056e280, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00046c6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000a71ef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00056e280, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00053c3a0, 0xc000420e40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 148 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00056e280, 0xc00001a780, 0xc000a127e0, 0xc0007d5260)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 164 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc000516280, 0xc00040ef00, 0xc00001a780, 0xc00072bc00, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:193 +0x351

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000516280, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:222 +0x17a

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d3ef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00072bef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000516280, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035ac30, 0xc00033f2a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 165 [runnable]:

encoding/json.(*decodeState).scanWhile(0xc000873ba8, 0x9)

	/usr/local/go/src/encoding/json/decode.go:347 +0xf0

encoding/json.(*decodeState).objectInterface(0xc000873ba8, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1132 +0x49

encoding/json.(*decodeState).valueInterface(0xc000873ba8, 0x9, 0x0)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).arrayInterface(0xc000873ba8, 0x6, 0x1cf1, 0x2ace)

	/usr/local/go/src/encoding/json/decode.go:1111 +0x9d

encoding/json.(*decodeState).valueInterface(0xc000873ba8, 0x9, 0xa)

	/usr/local/go/src/encoding/json/decode.go:1090 +0x4d

encoding/json.(*decodeState).objectInterface(0xc000873ba8, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).valueInterface(0xc000873ba8, 0x9, 0x6)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).objectInterface(0xc000873ba8, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).valueInterface(0xc000873ba8, 0x9, 0x3c)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).objectInterface(0xc000873ba8, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).valueInterface(0xc000873ba8, 0x9, 0x5)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).objectInterface(0xc000873ba8, 0xc000d4cf60)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).object(0xc000873ba8, 0x1708de0, 0xc000d4cf60, 0x194, 0xc000873bd0, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:679 +0x2036

encoding/json.(*decodeState).value(0xc000873ba8, 0x1708de0, 0xc000d4cf60, 0x194, 0xc000d4d460, 0x94)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).object(0xc000873ba8, 0x166af20, 0xc0002fa180, 0x16, 0xc000873bd0, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:823 +0x1392

encoding/json.(*decodeState).value(0xc000873ba8, 0x166af20, 0xc0002fa180, 0x16, 0xc000831318, 0x51574e)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).unmarshal(0xc000873ba8, 0x166af20, 0xc0002fa180, 0x0, 0x16b8001)

	/usr/local/go/src/encoding/json/decode.go:179 +0x1fb

encoding/json.(*Decoder).Decode(0xc000873b80, 0x166af20, 0xc0002fa180, 0x189aa80, 0xc000831301)

	/usr/local/go/src/encoding/json/stream.go:73 +0x166

k8s.io/apimachinery/pkg/util/json.Unmarshal(0xc000c82000, 0x2ace, 0x3000, 0x166af20, 0xc0002fa180, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/json/json.go:51 +0x2c1

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToUnstructured(0xc000c82000, 0x2ace, 0x3000, 0xc000831488, 0xc000d42580, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:416 +0x8c

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToList(0xc000bec000, 0x3282f, 0x3fe00, 0xc000610360, 0xc000610340, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:449 +0x31e

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decode(0xc000bec000, 0x3282f, 0x3fe00, 0x1, 0x1, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:387 +0xcd

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.Decode(0xc000bec000, 0x3282f, 0x3fe00, 0x0, 0x0, 0x0, 0x409eb5, 0x17566c0, 0x1825800, 0x7fbdc3eefd00, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:337 +0x2e2

k8s.io/apimachinery/pkg/runtime.Decode(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/codec.go:54

k8s.io/client-go/dynamic.(*dynamicResourceClient).List(0xc000316be0, 0x0, 0x0, 0x0, 0x0, 0xc00003e230, 0x50, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:269 +0x3d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:33 +0x1e9

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc000046028, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000831b98, 0x1bef580, 0xc000046028, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*ListWatch).List(0xc00033ece0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:106 +0x16e

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1.2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:184 +0x72

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc000046020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000831f10, 0x1bef580, 0xc000046020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1(0xc0007d4300, 0xc000516280, 0xc00040ef00, 0xc0006d6440, 0xc0006d6450, 0xc0000aad80)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:190 +0x1a4

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:175 +0x287



goroutine 61 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00062e180)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710410)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 62 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00082cdd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000736740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00082cf40, 0xdf8475800, 0x0, 0x1, 0xc0007444e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00062e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 63 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00062e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710430)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 64 [runnable]:

sync.runtime_SemacquireMutex(0xc00000c2a4, 0xc0000b5300, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc00000c2a0)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x17d8fe0, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:264 +0x584

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0009b8a20, 0x15, 0x17d8fe0, 0xc0002cd000, 0x199, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0009b8a20, 0x15, 0x17d8fe0, 0xc0002cd000, 0x199, 0x199, 0x15e0d23)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0009b8a20, 0x15, 0x1820100, 0xc0002cd000, 0x199, 0x7fbdc4163100, 0x20300000000000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:384 +0x1ba

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0009b8a20, 0x15, 0x1820100, 0xc0002cd000, 0x199, 0x7fbdc409d078, 0x4)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc0009b8a20, 0x1942160, 0xc0002cd000, 0x138a15b9fdcba5ab, 0xc0009b8ab0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000172648, 0xc000dc0000, 0xc000e8b3c0, 0x1f)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc00026a0f0, 0x193f940, 0xc000172648)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc0003880e0, 0x193f940, 0xc000172648)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0000b5dd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000737740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0000b5f40, 0xdf8475800, 0x0, 0x1, 0xc000744540)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc0007d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710440)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 177 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0007d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc000710450)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 167 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00040e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001566f0, 0xc0006d66a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 169 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00040e500)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001567d0, 0xc0006d66c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 170 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00083cdd8, 0x0, 0xc0006d35f8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d3740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00083cf40, 0xdf8475800, 0x0, 0x42db01, 0xc0007445a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00062e180)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d66d0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 171 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00062e180)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d66e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 173 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00062e200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d6700)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 174 [runnable]:

reflect.cvtDirect(0x16b8040, 0xc000e9cc50, 0x98, 0x1c2b8e0, 0x16b7400, 0x15c4000, 0x1, 0xc00070be10)

	/usr/local/go/src/reflect/value.go:2646 +0xcf

reflect.Value.Convert(0x16b8040, 0xc000e9cc50, 0x98, 0x1c2b8e0, 0x16b7400, 0x98, 0xc00070b538, 0x413b0f)

	/usr/local/go/src/reflect/value.go:2417 +0xda

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x16b8040, 0xc000e9cc50, 0x98, 0x16b7400, 0xc000dc8060, 0x198, 0x1708de0, 0xc000dd80f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:178 +0xb5b

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0009b8750, 0x15, 0x18f6340, 0xc000dc8020, 0x199, 0xc00070b6f8, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0009b8750, 0x15, 0x18f6340, 0xc000dc8020, 0x199, 0x1708de0, 0xc000dd8050)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0009b86f0, 0x15, 0x1820100, 0xc000dc8000, 0x199, 0x7fbdc42f4fff, 0x400)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0009b86f0, 0x15, 0x1820100, 0xc000dc8000, 0x199, 0x7fbdc40b6600, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc0009b86f0, 0x1942160, 0xc000dc8000, 0xadd28906eec41074, 0xc0009b8780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000172640, 0xc000dc4000, 0xc000e8b140, 0x1b)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc00026a0f0, 0x193f940, 0xc000172640)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc0003880e0, 0x193f940, 0xc000172640)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00070bdd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000540740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00070bf40, 0xdf8475800, 0x0, 0x1, 0xc0001b07e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc0007d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d6710)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 175 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0007d2000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00028c8b0, 0xc0006d6720)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 210 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0007fde80, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0007fde70)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc0007fde68, 0xc000a30200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc0007fde40, 0xc000a30200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00081c580, 0xc000758220, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00081c580, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00081c580, 0x1742800, 0xc000758220, 0xc00004adf8, 0xc00004a000)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000a201e0, 0xc000a34000, 0x400, 0x400, 0x0, 0x1, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000316050, 0x0, 0x1bbde40, 0xc000a320c0, 0xc0002fa0b0, 0x178d980, 0xc0006d3f58, 0x130338d, 0xc0006d3f08)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000758200, 0xc0006d3e90, 0xc000744060, 0x18, 0xc0006d3e20, 0x130326e, 0x1a07710)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000a32080)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 193 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00028c930, 0xc0000ab020)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035ad80, 0xc00033f620)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 194 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00001a780, 0xc00040f400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 195 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000516320, 0x1bbe340, 0xc0009dd7c0, 0xc000709c00, 0xc0004b5b60, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000516320, 0xc00001a780, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000473ef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000709ef0, 0x3b9aca00, 0x0, 0x1, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000516320, 0xc00001a780)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00035adc0, 0xc00033f660)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 197 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc000516320, 0xc00001a780, 0xc0006fdf80, 0xc0004b5b60)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 199 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0008725c0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0008725b0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc0008725a8, 0xc0008b6200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc000872580, 0xc0008b6200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00024b760, 0xc000933980, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00024b760, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00024b760, 0x1742800, 0xc000933980, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0006742a0, 0xc0000f5000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00026a960, 0x0, 0x1bbde40, 0xc00087b380, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000933960, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00087b340)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 150 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00024b4e0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00024b4d0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc00024b4c8, 0xc0003a3c00, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc00024b4a0, 0xc0003a3c00, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00083e420, 0xc000a1a460, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00083e420, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00083e420, 0x1742800, 0xc000a1a460, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000a14720, 0xc000a28000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000254f50, 0x0, 0x1bbde40, 0xc0009dd800, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000a1a440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0009dd7c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 83 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00056d380, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00056d370)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc00056d368, 0xc000e60000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc00056d340, 0xc000e60000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00081cb00, 0xc000550e00, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00081cb00, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00081cb00, 0x1742800, 0xc000550e00, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00049f410, 0xc000a34400, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00026a9b0, 0x0, 0x1bbde40, 0xc000db2700, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000550d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000db26c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe







time="2020-07-28T16:11:49Z" level=info msg="config map" name=workflow-controller-configmap

time="2020-07-28T16:11:49Z" level=info msg="Configuration:\nartifactRepository: {}\nmetricsConfig: {}\nnodeEvents: {}\npodSpecLogStrategy: {}\nsso:\n  clientId:\n    key: \"\"\n  clientSecret:\n    key: \"\"\n  issuer: \"\"\n  redirectUrl: \"\"\ntelemetryConfig: {}\n"

time="2020-07-28T16:11:49Z" level=info msg="Persistence configuration disabled"

time="2020-07-28T16:11:49Z" level=info msg="Starting Workflow Controller" version=v2.9.1

time="2020-07-28T16:11:49Z" level=info msg="Workers: workflow: 32, pod: 32"

time="2020-07-28T16:11:49Z" level=info msg="Performing periodic GC every 5m0s"

time="2020-07-28T16:11:49Z" level=info msg="Persistence disabled - so archived workflow GC disabled - you must restart the controller if you enable this"

time="2020-07-28T16:11:49Z" level=info msg="Starting workflow TTL controller (resync 20m0s)"

time="2020-07-28T16:11:49Z" level=info msg="Starting CronWorkflow controller"

time="2020-07-28T16:11:49Z" level=info msg="Starting prometheus metrics server at localhost:9090/metrics"

time="2020-07-28T16:11:49Z" level=info msg="Processing release/seo-impressions-workflow"

time="2020-07-28T16:11:49Z" level=info msg="Processing default/disco-3130-workflow"

fatal error: concurrent map read and map write



goroutine 137 [running]:

runtime.throw(0x197342d, 0x21)

	/usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc00052cbc8 sp=0xc00052cb98 pc=0x42e5d2

runtime.mapaccess1_faststr(0x170e600, 0xc0003d4e70, 0xc00050e1e0, 0x21, 0x194d89b)

	/usr/local/go/src/runtime/map_faststr.go:21 +0x44f fp=0xc00052cc38 sp=0xc00052cbc8 pc=0x41285f

github.com/argoproj/argo/workflow/metrics.(*Metrics).WorkflowAdded(0xc000165440, 0xc00050e1e0, 0x21, 0xc0001ea445, 0x9)

	/go/src/github.com/argoproj/argo/workflow/metrics/metrics.go:92 +0x56 fp=0xc00052cc78 sp=0xc00052cc38 pc=0x1495f56

github.com/argoproj/argo/workflow/controller.(*WorkflowController).addWorkflowInformerHandlers.func5(0x193f940, 0xc000cc2218)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:592 +0xbd fp=0xc00052ccd0 sp=0xc00052cc78 pc=0x15b4e6d

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd(0xc00000d6c0, 0x193f940, 0xc000cc2218)

	<autogenerated>:1 +0x5a fp=0xc00052ccf0 sp=0xc00052ccd0 pc=0x130b5da

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x6751656b6b587846, 0x6e376e2b35617863, 0x30617232442f696b)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218 fp=0xc00052cd68 sp=0xc00052ccf0 pc=0x13093d8

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00052cdd8, 0x786343574f4f4153, 0x6e6930554c707758)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51 fp=0xc00052cd90 sp=0xc00052cd68 pc=0x12c84b1

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79 fp=0xc00052cdf8 sp=0xc00052cd90 pc=0x1309499

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000595f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e fp=0xc00052ce68 sp=0xc00052cdf8 pc=0x12c8b6e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00052cf40, 0xdf8475800, 0x0, 0x55467a316c346301, 0xc0006229c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8 fp=0xc00052cf18 sp=0xc00052ce68 pc=0x12c8108

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b fp=0xc00052cf68 sp=0xc00052cf18 pc=0x13035db

k8s.io/client-go/tools/cache.(*processorListener).run-fm()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:593 +0x2a fp=0xc00052cf80 sp=0xc00052cf68 pc=0x130b2da

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0007327f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59 fp=0xc00052cfd0 sp=0xc00052cf80 pc=0x12c8ae9

runtime.goexit()

	/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00052cfd8 sp=0xc00052cfd0 pc=0x45b741

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 1 [select (no cases)]:

main.NewRootCommand.func1(0xc00012ea00, 0xc000376ec0, 0x0, 0x4, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:89 +0x3f4

github.com/spf13/cobra.(*Command).execute(0xc00012ea00, 0xc0000da010, 0x4, 0x4, 0xc00012ea00, 0xc0000da010)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:762 +0x460

github.com/spf13/cobra.(*Command).ExecuteC(0xc00012ea00, 0xc00006a750, 0xc000571f50, 0x40576f)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:852 +0x2ea

github.com/spf13/cobra.(*Command).Execute(...)

	/go/pkg/mod/github.com/spf13/[email protected]/command.go:800

main.main()

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:111 +0x2b



goroutine 18 [chan receive]:

k8s.io/klog.(*loggingT).flushDaemon(0x288fb80)

	/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b

created by k8s.io/klog.init.0

	/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd6



goroutine 19 [syscall]:

os/signal.signal_recv(0x0)

	/usr/local/go/src/runtime/sigqueue.go:147 +0x9c

os/signal.loop()

	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22

created by os/signal.init.0

	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41



goroutine 7 [chan receive]:

github.com/argoproj/pkg/stats.RegisterStackDumper.func1()

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:31 +0xa0

created by github.com/argoproj/pkg/stats.RegisterStackDumper

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:27 +0x35



goroutine 8 [chan receive]:

github.com/argoproj/pkg/stats.StartStatsTicker.func1(0xc00003a370)

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:19 +0x37

created by github.com/argoproj/pkg/stats.StartStatsTicker

	/go/pkg/mod/github.com/argoproj/[email protected]/stats/stats_linux.go:17 +0x4d



goroutine 9 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00054c0c0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 10 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc00012bf80)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 11 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00054c1e0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 12 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000146000)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 28 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00073a2a0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 152 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0bd0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 56 [runnable]:

internal/poll.runtime_pollWait(0x7f801769ffd0, 0x72, 0xffffffffffffffff)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc0002bef98, 0x72, 0x8d00, 0x8dfa, 0xffffffffffffffff)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Read(0xc0002bef80, 0xc0007b0000, 0x8dfa, 0x8dfa, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf

net.(*netFD).Read(0xc0002bef80, 0xc0007b0000, 0x8dfa, 0x8dfa, 0x203000, 0x0, 0x8ded)

	/usr/local/go/src/net/fd_unix.go:202 +0x4f

net.(*conn).Read(0xc00000e010, 0xc0007b0000, 0x8dfa, 0x8dfa, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/net.go:184 +0x68

crypto/tls.(*atLeastReader).Read(0xc0006f40a0, 0xc0007b0000, 0x8dfa, 0x8dfa, 0x7a7, 0x1001bab3c0, 0xc000571970)

	/usr/local/go/src/crypto/tls/conn.go:780 +0x60

bytes.(*Buffer).ReadFrom(0xc000094958, 0x1bab100, 0xc0006f40a0, 0x409eb5, 0x1755ec0, 0x18e2f80)

	/usr/local/go/src/bytes/buffer.go:204 +0xb4

crypto/tls.(*Conn).readFromUntil(0xc000094700, 0x1bacec0, 0xc00000e010, 0x5, 0xc00000e010, 0x203000)

	/usr/local/go/src/crypto/tls/conn.go:802 +0xec

crypto/tls.(*Conn).readRecordOrCCS(0xc000094700, 0x0, 0x0, 0xc0006f4020)

	/usr/local/go/src/crypto/tls/conn.go:609 +0x124

crypto/tls.(*Conn).readRecord(...)

	/usr/local/go/src/crypto/tls/conn.go:577

crypto/tls.(*Conn).Read(0xc000094700, 0xc000693000, 0x1000, 0x1000, 0x0, 0x0, 0x0)

	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161

bufio.(*Reader).Read(0xc0000a9800, 0xc000694038, 0x9, 0x9, 0xc000571d10, 0x0, 0x8a2bb5)

	/usr/local/go/src/bufio/bufio.go:226 +0x26a

io.ReadAtLeast(0x1baaee0, 0xc0000a9800, 0xc000694038, 0x9, 0x9, 0x9, 0xc0000ac100, 0x0, 0x1bab3c0)

	/usr/local/go/src/io/io.go:310 +0x87

io.ReadFull(...)

	/usr/local/go/src/io/io.go:329

golang.org/x/net/http2.readFrameHeader(0xc000694038, 0x9, 0x9, 0x1baaee0, 0xc0000a9800, 0x0, 0x0, 0xc00088e0f0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x87

golang.org/x/net/http2.(*Framer).ReadFrame(0xc000694000, 0xc00088e0f0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0xa1

golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000571fb8, 0xc00046afa8, 0x88bb98)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1746 +0x8c

golang.org/x/net/http2.(*ClientConn).readLoop(0xc000482600)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1674 +0xa3

created by golang.org/x/net/http2.(*Transport).newClientConn

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:674 +0x62f



goroutine 15 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc000436c00, 0xc00056bd18, 0xc000622480, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc00056bd18, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc000622420, 0xc00056be78, 0x1, 0x1, 0xc000048c80)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run(0xc000480e00, 0x1bef540, 0xc000048c80, 0x20, 0x20)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:163 +0x8f3

created by main.NewRootCommand.func1

	/go/src/github.com/argoproj/argo/cmd/workflow-controller/main.go:86 +0x3ef



goroutine 16 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00018a0d8, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00018a0c8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00018a0b0, 0xc00021ac30, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00011f680)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00044cec8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00044cec8, 0x3b9aca00, 0x0, 0xc000668301, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00011f680, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

github.com/argoproj/argo/config.(*controller).Run(0xc0002c33b0, 0xc000622420, 0xc000732360)

	/go/src/github.com/argoproj/argo/config/controller.go:119 +0x384

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:148 +0x44c



goroutine 97 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00018a188, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00018a178)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00018a160, 0xc0003e06a0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000418f00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005cae00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005cae00, 0x3b9aca00, 0x0, 0xc000741501, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000418f00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012cb40, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:149 +0x4a2



goroutine 98 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00018a028, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00018a018)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00018a000, 0xc0003e0010, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00011e000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00044ae00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00044ae00, 0x3b9aca00, 0x0, 0xc0000ba601, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc00011e000, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012cc60, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:150 +0x52c



goroutine 99 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00066a028, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00066a018)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00066a000, 0xc0003a6050, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000572000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00044fe00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00044fe00, 0x3b9aca00, 0x0, 0xc000122c01, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000572000, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012cbd0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:151 +0x582



goroutine 100 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podLabeler(0xc000480e00, 0xc000622420)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:215 +0xda

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:152 +0x5cb



goroutine 101 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).podGarbageCollector(0xc000480e00, 0xc000622420)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:241 +0xd4

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:153 +0x614



goroutine 102 [select]:

github.com/argoproj/argo/workflow/controller.(*WorkflowController).workflowGarbageCollector(0xc000480e00, 0xc000622420)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:275 +0x18e

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:154 +0x65d



goroutine 109 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0000d5080)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 104 [select]:

k8s.io/apimachinery/pkg/util/wait.WaitFor(0xc000436ce0, 0xc0006fee40, 0xc0006226c0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:432 +0x12e

k8s.io/apimachinery/pkg/util/wait.PollUntil(0x5f5e100, 0xc00063de40, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:385 +0xd6

k8s.io/client-go/tools/cache.WaitForCacheSync(0xc000622420, 0xc00063df28, 0x1, 0x1, 0xc000619f50)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:161 +0x7c

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run(0xc00003a460, 0xc000622420, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:63 +0x211

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runTTLController(0xc000480e00, 0x1bef540, 0xc000048c80)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:123 +0x86

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:157 +0x6e5



goroutine 105 [chan receive]:

github.com/argoproj/argo/workflow/cron.(*Controller).Run(0xc000508000, 0x1bef540, 0xc000048c80)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:112 +0x66a

github.com/argoproj/argo/workflow/controller.(*WorkflowController).runCronController(0xc000480e00, 0x1bef540, 0xc000048c80)

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:131 +0xc0

created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run

	/go/src/github.com/argoproj/argo/workflow/controller/controller.go:158 +0x724



goroutine 29 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000147880)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 107 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc000622420, 0xc000732390, 0x1bef540, 0xc000049000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 108 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc000622540, 0x5f5e100, 0x0, 0xc0006224e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 26 [chan receive]:

github.com/argoproj/argo/workflow/metrics.runServer(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc0006140a0, 0x1bef540, 0xc000048c80)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:59 +0x291

created by github.com/argoproj/argo/workflow/metrics.(*Metrics).RunServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:36 +0x28f



goroutine 114 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00012a070, 0xc0000aa540)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00017e000, 0xc0003f41a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 115 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc000572000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 59 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00012a0e0, 0xc000506000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea000, 0xc0003ac040)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 60 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc00011e000)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 110 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc00012a380)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 116 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0001348c0, 0x1bbe340, 0xc0005cd940, 0xc000717c00, 0xc0000a8de0, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0001348c0, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00006cef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000717ef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0001348c0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00017e020, 0xc0003f4380)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 61 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00029d860, 0x1bbe340, 0xc00065a5c0, 0xc0005bbc00, 0xc00054db00, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00029d860, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00046bef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005bbef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00029d860, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea020, 0xc0003ac0a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 180 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0001348c0, 0xc000622420, 0xc0006334a0, 0xc0000a8de0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 173 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00029d860, 0xc000622420, 0xc000a05f80, 0xc00054db00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 111 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).add(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:560

k8s.io/client-go/tools/cache.(*sharedProcessor).distribute(0xc0000dfe30, 0x177f0a0, 0xc000962890, 0xc000962801)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:459 +0xe7

k8s.io/client-go/tools/cache.(*sharedIndexInformer).HandleDeltas(0xc00012cb40, 0x1757640, 0xc000780fa0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:416 +0x2ab

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001806e0, 0xc000732520, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:422 +0x230

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000501e00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00061be00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00061be00, 0x3b9aca00, 0x0, 0xc000741501, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000501e00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00012cb40, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:62 +0x17d



goroutine 112 [select]:

k8s.io/apimachinery/pkg/util/wait.contextForChannel.func1(0xc000622420, 0xc000732500, 0x1bef540, 0xc000049140)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:271 +0xb7

created by k8s.io/apimachinery/pkg/util/wait.contextForChannel

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:270 +0x8c



goroutine 129 [select]:

k8s.io/apimachinery/pkg/util/wait.poller.func1.1(0xc000622780, 0x5f5e100, 0x0, 0xc000622720)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:481 +0x18e

created by k8s.io/apimachinery/pkg/util/wait.poller.func1

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:464 +0x8c



goroutine 135 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xbfc031714c0428f9, 0x42c437b)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00052add8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000068f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00052af40, 0xdf8475800, 0x0, 0x42db01, 0xc000622960)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0007327d0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 131 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0000dfe30, 0xc000622840)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0003ff930, 0xc00000dba0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 132 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc000501e00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 133 [select]:

golang.org/x/net/http2.(*ClientConn).roundTrip(0xc000482600, 0xc001131100, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1096 +0x9a0

golang.org/x/net/http2.(*Transport).RoundTripOpt(0xc0002ae060, 0xc001131100, 0xc000d5ed00, 0x754025, 0x0, 0xc00018d200)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:448 +0x172

golang.org/x/net/http2.(*Transport).RoundTrip(...)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:409

golang.org/x/net/http2.noDialH2RoundTripper.RoundTrip(0xc0002ae060, 0xc001131100, 0xc0006666e0, 0x5, 0xc00018d288)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2613 +0x3e

net/http.(*Transport).roundTrip(0xc0004aca00, 0xc001131100, 0x16c7600, 0xc000522701, 0xc000b649a0)

	/usr/local/go/src/net/http/transport.go:485 +0xdc1

net/http.(*Transport).RoundTrip(0xc0004aca00, 0xc001131100, 0x1954ebc, 0xd, 0xc000b64a58)

	/usr/local/go/src/net/http/roundtrip.go:17 +0x35

k8s.io/client-go/transport.(*bearerAuthRoundTripper).RoundTrip(0xc0003d5470, 0xc001131000, 0x1951e42, 0xa, 0xc000b648e0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:317 +0x266

k8s.io/client-go/transport.(*userAgentRoundTripper).RoundTrip(0xc00000d420, 0xc001130f00, 0xc00000d420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:167 +0x1c0

net/http.send(0xc001130f00, 0x1bacd40, 0xc00000d420, 0x0, 0x0, 0x0, 0xc00086a100, 0x5c8036, 0x1, 0x0)

	/usr/local/go/src/net/http/client.go:250 +0x443

net/http.(*Client).send(0xc0003d54a0, 0xc001130f00, 0x0, 0x0, 0x0, 0xc00086a100, 0x0, 0x1, 0xc00078c8a0)

	/usr/local/go/src/net/http/client.go:174 +0xfa

net/http.(*Client).do(0xc0003d54a0, 0xc001130f00, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/http/client.go:641 +0x3ce

net/http.(*Client).Do(0xc0003d54a0, 0xc001130f00, 0x0, 0x3, 0xc0006666e0)

	/usr/local/go/src/net/http/client.go:509 +0x35

k8s.io/client-go/rest.(*Request).WatchWithSpecificDecoders(0xc00076e180, 0xc0006ed7c8, 0x7f80173db008, 0x28ad300, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:573 +0x1e2

k8s.io/client-go/dynamic.(*dynamicResourceClient).Watch(0xc00003a5a0, 0x0, 0x0, 0x0, 0x0, 0xc000bef260, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:309 +0x8d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000c310c8, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:39 +0x1e9

k8s.io/client-go/tools/cache.(*ListWatch).Watch(0xc00000d520, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:113 +0x6c

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0000ecdc0, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:275 +0x420

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005466f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006edef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0000ecdc0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0003ff950, 0xc00000dbe0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 174 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0000ecdc0, 0xc000622420, 0xc000d7e1e0, 0xc0002ae2a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 30 [chan receive]:

k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00073a3c0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0

created by k8s.io/client-go/util/workqueue.newQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132



goroutine 31 [select]:

k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc0001479d0)

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:215 +0x344

created by k8s.io/client-go/util/workqueue.newDelayingQueue

	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:56 +0x1d4



goroutine 63 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc00011f680)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 136 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0007327e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 138 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc000732800)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 139 [semacquire]:

sync.runtime_SemacquireMutex(0xc0000ca264, 0xc000bfa901, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc0000ca260)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x18d7f40, 0x3, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:264 +0x584

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0008c8360, 0x15, 0x18d7f40, 0xc000d54380, 0x199, 0x6f69746172754473, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000d83dc0, 0x194, 0x18d7f40, 0xc000d54380, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc0007d6b00, 0x97, 0x167a6c0, 0xc0006bcac0, 0x197, 0xc000be6e28, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc0007d6b00, 0x97, 0x167a6c0, 0xc0006bcac0, 0x197, 0x1708de0, 0xc000962680)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0008c8330, 0x15, 0x17e6ac0, 0xc0006bcab0, 0x199, 0x4a6c29, 0x17e6ac0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0008c8330, 0x15, 0x17e6ac0, 0xc0006bcab0, 0x199, 0x10, 0x1708de0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.pointerFromUnstructured(0x170f560, 0xc0008c8330, 0x15, 0x189e220, 0xc00028ea20, 0x196, 0xc000be7180, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:368 +0x2b4

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0008c8330, 0x15, 0x189e220, 0xc00028ea20, 0x196, 0x1708de0, 0xc000962660)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:226 +0x701

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0008c82d0, 0x15, 0x193c3e0, 0xc00028e900, 0x199, 0x0, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000c8f820, 0x194, 0x193c3e0, 0xc00028e900, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc0007d6b20, 0x97, 0x167a900, 0xc000122118, 0x197, 0xc000be7538, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc0007d6b20, 0x97, 0x167a900, 0xc000122118, 0x197, 0x1708de0, 0xc0009625f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0008c81b0, 0x15, 0x1923380, 0xc000122118, 0x199, 0xc000be76f8, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0008c81b0, 0x15, 0x1923380, 0xc000122118, 0x199, 0x1708de0, 0xc0009625e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc0008c8090, 0x15, 0x1820100, 0xc000122000, 0x199, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc0008c8090, 0x15, 0x1820100, 0xc000122000, 0x199, 0x7f80176a2600, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc0008c8090, 0x1942160, 0xc000122000, 0x6bd522f6c27203cc, 0xc0008c8150)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000cc2148, 0xc0006bca20, 0xc000e6a160, 0x1f)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc00003a460, 0x193f940, 0xc000cc2148)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc00000da00, 0x193f940, 0xc000cc2148)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000bfbdd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000498f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000bfbf40, 0xdf8475800, 0x0, 0x1, 0xc000622a20)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501a00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc000732810)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 140 [runnable]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501a00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc000732820)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 64 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00029d9a0, 0x1bbe340, 0xc0008684c0, 0xc00063dc00, 0xc00054c720, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00029d9a0, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0004696f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00063def0, 0x3b9aca00, 0x0, 0x192ee01, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00029d9a0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea120, 0xc0003ac8e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 37 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00029d9a0, 0xc000622420, 0xc0007780c0, 0xc00054c720)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 141 [IO wait]:

internal/poll.runtime_pollWait(0x7f801769ff00, 0x72, 0x0)

	/usr/local/go/src/runtime/netpoll.go:184 +0x55

internal/poll.(*pollDesc).wait(0xc0003e7318, 0x72, 0x0, 0x0, 0x194ee55)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45

internal/poll.(*pollDesc).waitRead(...)

	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92

internal/poll.(*FD).Accept(0xc0003e7300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8

net.(*netFD).accept(0xc0003e7300, 0xc0005b9d00, 0xc00070c000, 0x7f801990c008)

	/usr/local/go/src/net/fd_unix.go:238 +0x42

net.(*TCPListener).accept(0xc00000df60, 0xc0005b9d30, 0x40c748, 0x30)

	/usr/local/go/src/net/tcpsock_posix.go:139 +0x32

net.(*TCPListener).Accept(0xc00000df60, 0x1842ec0, 0xc0004f0060, 0x1721a80, 0x286f000)

	/usr/local/go/src/net/tcpsock.go:261 +0x47

net/http.(*Server).Serve(0xc0006940e0, 0x1be9500, 0xc00000df60, 0x0, 0x0)

	/usr/local/go/src/net/http/server.go:2896 +0x286

net/http.(*Server).ListenAndServe(0xc0006940e0, 0x4, 0x199a4ef)

	/usr/local/go/src/net/http/server.go:2825 +0xb7

github.com/argoproj/argo/workflow/metrics.runServer.func1(0x1, 0x194f52b, 0x8, 0x194c64a, 0x4, 0x0, 0x0, 0xc0006940e0)

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:53 +0xf0

created by github.com/argoproj/argo/workflow/metrics.runServer

	/go/src/github.com/argoproj/argo/workflow/metrics/server.go:51 +0x260



goroutine 151 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000529dd8, 0x0, 0xc0004695f8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000469740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000529f40, 0xdf8475800, 0x0, 0x42db01, 0xc000506480)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501400)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0ba0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 147 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0000dfe30, 0xc000506360)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea220, 0xc0003acd20)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 148 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc000418f00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 149 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc00029dae0, 0xc000419700, 0xc000622420, 0xc0005cbc00, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:193 +0x351

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00029dae0, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:222 +0x17a

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00046b6f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005cbef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00029dae0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea240, 0xc0003acec0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 150 [runnable]:

encoding/json.(*decodeState).scanWhile(0xc0004a1628, 0x9)

	/usr/local/go/src/encoding/json/decode.go:347 +0xf0

encoding/json.(*decodeState).objectInterface(0xc0004a1628, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1157 +0x15f

encoding/json.(*decodeState).valueInterface(0xc0004a1628, 0x9, 0x9)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).objectInterface(0xc0004a1628, 0x2)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).valueInterface(0xc0004a1628, 0x9, 0x0)

	/usr/local/go/src/encoding/json/decode.go:1093 +0xb0

encoding/json.(*decodeState).arrayInterface(0xc0004a1628, 0x6, 0x227, 0x6b0)

	/usr/local/go/src/encoding/json/decode.go:1111 +0x9d

encoding/json.(*decodeState).valueInterface(0xc0004a1628, 0x9, 0x9)

	/usr/local/go/src/encoding/json/decode.go:1090 +0x4d

encoding/json.(*decodeState).objectInterface(0xc0004a1628, 0xc000c9f9c0)

	/usr/local/go/src/encoding/json/decode.go:1160 +0x170

encoding/json.(*decodeState).object(0xc0004a1628, 0x1708de0, 0xc000c9f9c0, 0x194, 0xc0004a1650, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:679 +0x2036

encoding/json.(*decodeState).value(0xc0004a1628, 0x1708de0, 0xc000c9f9c0, 0x194, 0xc000e96080, 0x94)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).object(0xc0004a1628, 0x166af20, 0xc0005601b0, 0x16, 0xc0004a1650, 0x7b)

	/usr/local/go/src/encoding/json/decode.go:823 +0x1392

encoding/json.(*decodeState).value(0xc0004a1628, 0x166af20, 0xc0005601b0, 0x16, 0xc000629318, 0x51574e)

	/usr/local/go/src/encoding/json/decode.go:429 +0x6d

encoding/json.(*decodeState).unmarshal(0xc0004a1628, 0x166af20, 0xc0005601b0, 0x0, 0x16b8001)

	/usr/local/go/src/encoding/json/decode.go:179 +0x1fb

encoding/json.(*Decoder).Decode(0xc0004a1600, 0x166af20, 0xc0005601b0, 0x189aa80, 0xc000629301)

	/usr/local/go/src/encoding/json/stream.go:73 +0x166

k8s.io/apimachinery/pkg/util/json.Unmarshal(0xc000a74e00, 0x6b0, 0x700, 0x166af20, 0xc0005601b0, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/json/json.go:51 +0x2c1

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToUnstructured(0xc000a74e00, 0x6b0, 0x700, 0xc000629488, 0xc00122ba98, 0x8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:416 +0x8c

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decodeToList(0xc000910000, 0x383fe, 0x3fe00, 0xc000888240, 0xc000888220, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:449 +0x31e

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.decode(0xc000910000, 0x383fe, 0x3fe00, 0x1, 0x1, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:387 +0xcd

k8s.io/apimachinery/pkg/apis/meta/v1/unstructured.unstructuredJSONScheme.Decode(0xc000910000, 0x383fe, 0x3fe00, 0x0, 0x0, 0x0, 0x409eb5, 0x17566c0, 0x1825800, 0x7f80174fa300, ...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/unstructured/helpers.go:337 +0x2e2

k8s.io/apimachinery/pkg/runtime.Decode(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/codec.go:54

k8s.io/client-go/dynamic.(*dynamicResourceClient).List(0xc00062e050, 0x0, 0x0, 0x0, 0x0, 0xc0003f03f0, 0x2c, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:269 +0x3d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:33 +0x1e9

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000629b98, 0x1bef580, 0xc0000be008, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*ListWatch).List(0xc00000d520, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:106 +0x16e

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1.2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x194bc17, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:184 +0x72

k8s.io/client-go/tools/pager.SimplePageFunc.func1(0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:40 +0x64

k8s.io/client-go/tools/pager.(*ListPager).List(0xc000629f10, 0x1bef580, 0xc0000be000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/pager/pager.go:88 +0x12d

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1.1(0xc000624540, 0xc00029dae0, 0xc000419700, 0xc0003e0780, 0xc0003e0790, 0xc000506420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:190 +0x1a4

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:175 +0x287



goroutine 32 [select]:

github.com/robfig/cron/v3.(*Cron).run(0xc0006960a0)

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:263 +0x541

created by github.com/robfig/cron/v3.(*Cron).Start

	/go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:222 +0xa8



goroutine 161 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0005ea028, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0005ea018)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0005ea000, 0xc0006c63a0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000494800)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000665e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000665e00, 0x3b9aca00, 0x0, 0xc00014ea01, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000494800, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0006f8000, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:101 +0x455



goroutine 162 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00018a2e8, 0x1)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00018a2d8)

	/usr/local/go/src/sync/cond.go:56 +0x9d

k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00018a2c0, 0xc0003e0c70, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:409 +0xa6

k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000450880)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:150 +0x40

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006d5e00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d5e00, 0x3b9aca00, 0x0, 0xc0000d3101, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*controller).Run(0xc000450880, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:124 +0x2fd

k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0006f81b0, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:280 +0x47f

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:102 +0x4a5



goroutine 163 [semacquire]:

sync.runtime_SemacquireMutex(0xc0004e401c, 0xc0004a6f00, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc0004e4018)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc000508000, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:133 +0xb07

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc000508000)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006c6350)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006c6350, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006c6350, 0x3b9aca00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 164 [select]:

golang.org/x/net/http2.(*ClientConn).roundTrip(0xc000482600, 0xc001191800, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1096 +0x9a0

golang.org/x/net/http2.(*Transport).RoundTripOpt(0xc0002ae060, 0xc001191800, 0xc000da0a00, 0x754025, 0x0, 0xc00018d200)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:448 +0x172

golang.org/x/net/http2.(*Transport).RoundTrip(...)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:409

golang.org/x/net/http2.noDialH2RoundTripper.RoundTrip(0xc0002ae060, 0xc001191800, 0xc000e12690, 0x5, 0xc00018d288)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2613 +0x3e

net/http.(*Transport).roundTrip(0xc0004aca00, 0xc001191800, 0x16c7600, 0xc000864701, 0xc000634dc0)

	/usr/local/go/src/net/http/transport.go:485 +0xdc1

net/http.(*Transport).RoundTrip(0xc0004aca00, 0xc001191800, 0x1954ebc, 0xd, 0xc000634e78)

	/usr/local/go/src/net/http/roundtrip.go:17 +0x35

k8s.io/client-go/transport.(*bearerAuthRoundTripper).RoundTrip(0xc0002c2420, 0xc001191700, 0x1951e42, 0xa, 0xc000634d00)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:317 +0x266

k8s.io/client-go/transport.(*userAgentRoundTripper).RoundTrip(0xc0004aafc0, 0xc001191600, 0xc0004aafc0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:167 +0x1c0

net/http.send(0xc001191600, 0x1bacd40, 0xc0004aafc0, 0x0, 0x0, 0x0, 0xc000740d78, 0xc000a78900, 0x1, 0x0)

	/usr/local/go/src/net/http/client.go:250 +0x443

net/http.(*Client).send(0xc0002c24b0, 0xc001191600, 0x0, 0x0, 0x0, 0xc000740d78, 0x0, 0x1, 0x3f)

	/usr/local/go/src/net/http/client.go:174 +0xfa

net/http.(*Client).do(0xc0002c24b0, 0xc001191600, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/http/client.go:641 +0x3ce

net/http.(*Client).Do(0xc0002c24b0, 0xc001191600, 0x0, 0x3, 0xc000e12690)

	/usr/local/go/src/net/http/client.go:509 +0x35

k8s.io/client-go/rest.(*Request).request(0xc0003c3080, 0xc000da1378, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:737 +0x367

k8s.io/client-go/rest.(*Request).Do(0xc0003c3080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:809 +0xc5

github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1.(*workflowTemplates).Get(0xc000a7ed80, 0xc0005fb860, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/src/github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1/workflowtemplate.go:57 +0x192

github.com/argoproj/argo/workflow/templateresolution.(*workflowTemplateInterfaceWrapper).Get(0xc000b4b770, 0xc0005fb860, 0x19, 0x7, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:30 +0x72

github.com/argoproj/argo/workflow/templateresolution.(*Context).WithWorkflowTemplate(0xc00075d6c0, 0xc0005fb860, 0x19, 0x0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:282 +0x4b

github.com/argoproj/argo/workflow/templateresolution.(*Context).WithTemplateHolder(0xc00075d6c0, 0x1bb8900, 0xc000dd4460, 0x1, 0x1, 0x1bb8900)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:269 +0xa8

github.com/argoproj/argo/workflow/templateresolution.(*Context).resolveTemplateImpl(0xc00075d6c0, 0x1bb8900, 0xc000dd4460, 0x0, 0xe0, 0xd0, 0x18d7f40, 0xc0009d7b90, 0x1b823b4)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:233 +0x3a3

github.com/argoproj/argo/workflow/templateresolution.(*Context).ResolveTemplate(...)

	/go/src/github.com/argoproj/argo/workflow/templateresolution/context.go:180

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00075d500, 0x1bb8900, 0xc000dd4460, 0xc00075d6c0, 0x1bb8e00, 0x28ad5e0, 0xc000da1b30, 0x535, 0xe, 0xc0001d1500)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:496 +0xe2

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateDAG(0xc00075d500, 0xc0009d75c0, 0xc00075d6c0, 0xc000293d40, 0xc00028a0d8, 0xc0006f3500)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:1185 +0x4a5

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplate(0xc00075d500, 0xc0002926c0, 0xc00075d6c0, 0x1bb87c0, 0xc0009d7380, 0xc0006e8bf0, 0x18a4d00, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:427 +0xc40

github.com/argoproj/argo/workflow/validate.(*templateValidationCtx).validateTemplateHolder(0xc00075d500, 0x1bb8a40, 0xc000ce0420, 0xc0006e8d80, 0x1bb87c0, 0xc0009d7380, 0xc0006e8bf0, 0x0, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:518 +0x169

github.com/argoproj/argo/workflow/validate.ValidateWorkflow(0x1bab600, 0xc000b4b770, 0x1bab5e0, 0xc000b4b790, 0xc0002fb800, 0xc000b66f00, 0x0, 0x0, 0x0, 0x179, ...)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:234 +0x13c8

github.com/argoproj/argo/workflow/validate.ValidateCronWorkflow(0x1bab600, 0xc000b4b770, 0x1bab5e0, 0xc000b4b790, 0xc000871800, 0x0, 0x0)

	/go/src/github.com/argoproj/argo/workflow/validate/validate.go:317 +0x1a8

github.com/argoproj/argo/workflow/cron.(*cronWfOperationCtx).validateCronWorkflow(0xc0002b3b90, 0x1bb8b00, 0xc0003e1b70)

	/go/src/github.com/argoproj/argo/workflow/cron/operator.go:96 +0x178

github.com/argoproj/argo/workflow/cron.(*Controller).processNextCronItem(0xc000508000, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:152 +0x438

github.com/argoproj/argo/workflow/cron.(*Controller).runCronWorker(0xc000508000)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:116 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006c6360)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006c6360, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006c6360, 0x3b9aca00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:105 +0x4dc



goroutine 165 [semacquire]:

sync.runtime_SemacquireMutex(0xc0000ca264, 0xc00062c900, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc0000ca260)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x18d7f40, 0x8, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:264 +0x584

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000a65b00, 0x15, 0x18d7f40, 0xc0009d2e00, 0x199, 0xc000bda520, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000a6ecc0, 0x194, 0x18d7f40, 0xc0009d2e00, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc0007eda40, 0x97, 0x167a6c0, 0xc0007262b0, 0x197, 0xc00063ae38, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc0007eda40, 0x97, 0x167a6c0, 0xc0007262b0, 0x197, 0x1708de0, 0xc00075a8a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000a65aa0, 0x15, 0x17e6ac0, 0xc0007262a0, 0x199, 0x4a6c29, 0x17e6ac0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000a65aa0, 0x15, 0x17e6ac0, 0xc0007262a0, 0x199, 0x10, 0x1708de0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.pointerFromUnstructured(0x170f560, 0xc000a65aa0, 0x15, 0x189e220, 0xc0003ccea0, 0x196, 0xc00063b190, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:368 +0x2b4

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000a65aa0, 0x15, 0x189e220, 0xc0003ccea0, 0x196, 0x1708de0, 0xc00075a860)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:226 +0x701

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000a659e0, 0x15, 0x193c3e0, 0xc0003ccd80, 0x199, 0xc000736c10, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000c9f020, 0x194, 0x193c3e0, 0xc0003ccd80, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc0007eda60, 0x97, 0x167a900, 0xc0002fa918, 0x197, 0xc00063b548, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc0007eda60, 0x97, 0x167a900, 0xc0002fa918, 0x197, 0x1708de0, 0xc0009621b0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000a658f0, 0x15, 0x1923380, 0xc0002fa918, 0x199, 0xc00063b708, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000a658f0, 0x15, 0x1923380, 0xc0002fa918, 0x199, 0x1708de0, 0xc000962180)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc000a65800, 0x15, 0x1820100, 0xc0002fa800, 0x199, 0x0, 0x203000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc000a65800, 0x15, 0x1820100, 0xc0002fa800, 0x199, 0x7f80176a3700, 0x2)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc000a65800, 0x1942160, 0xc0002fa800, 0xc00050e000, 0x21)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000508180, 0xc00050e000, 0x21, 0x193f940)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc000508000, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:220 +0x2be

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc000508000)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006c6370)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006c6370, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006c6370, 0x3b9aca00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 166 [runnable]:

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x17e6ac0, 0x1, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:269 +0x42a

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00041f8f0, 0x15, 0x17e6ac0, 0xc0006bc960, 0x199, 0x4a6c29, 0x17e6ac0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00041f8f0, 0x15, 0x17e6ac0, 0xc0006bc960, 0x199, 0x10, 0x1708de0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.pointerFromUnstructured(0x170f560, 0xc00041f8f0, 0x15, 0x189e220, 0xc000b96a20, 0x196, 0xc000d61190, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:368 +0x2b4

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00041f8f0, 0x15, 0x189e220, 0xc000b96a20, 0x196, 0x1708de0, 0xc000962360)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:226 +0x701

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00041f860, 0x15, 0x193c3e0, 0xc000b96900, 0x199, 0xc0004eca80, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000a6ed80, 0x194, 0x193c3e0, 0xc000b96900, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc000790560, 0x97, 0x167a900, 0xc0007a6518, 0x197, 0xc000d61548, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc000790560, 0x97, 0x167a900, 0xc0007a6518, 0x197, 0x1708de0, 0xc00075a760)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00041f770, 0x15, 0x1923380, 0xc0007a6518, 0x199, 0xc000d61708, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00041f770, 0x15, 0x1923380, 0xc0007a6518, 0x199, 0x1708de0, 0xc00075a750)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00041f680, 0x15, 0x1820100, 0xc0007a6400, 0x199, 0x203000, 0xc0000de000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00041f680, 0x15, 0x1820100, 0xc0007a6400, 0x199, 0x7f80174b22f0, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc00041f680, 0x1942160, 0xc0007a6400, 0xc000a5e000, 0x47)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000508188, 0xc000a5e000, 0x47, 0x193f940)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/cron.(*Controller).processNextWorkflowItem(0xc000508000, 0x0)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:220 +0x2be

github.com/argoproj/argo/workflow/cron.(*Controller).runWorkflowWorker(0xc000508000)

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:188 +0x2b

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006c6380)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006c6380, 0x3b9aca00, 0x0, 0x544e314d544d3201, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(0xc0006c6380, 0x3b9aca00, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d

created by github.com/argoproj/argo/workflow/cron.(*Controller).Run

	/go/src/github.com/argoproj/argo/workflow/cron/controller.go:109 +0x5a6



goroutine 143 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000494200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000147f70, 0xc000732980)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 168 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000147d50, 0xc0006324e0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0004e4060, 0xc0001fdfe0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 169 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc000494800)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 153 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000661dd8, 0x0, 0xc00046b5f8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00046b740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000661f40, 0xdf8475800, 0x0, 0x1, 0xc0005064e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0c00)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 154 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501600)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0c10)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 155 [semacquire]:

sync.runtime_SemacquireMutex(0xc0000ca264, 0xc0005bec00, 0x1)

	/usr/local/go/src/runtime/sema.go:71 +0x47

sync.(*Mutex).lockSlow(0xc0000ca260)

	/usr/local/go/src/sync/mutex.go:138 +0xfc

sync.(*Mutex).Lock(...)

	/usr/local/go/src/sync/mutex.go:81

k8s.io/apimachinery/pkg/runtime.fieldInfoFromField(0x1c2b8e0, 0x19119a0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:264 +0x584

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00089be90, 0x15, 0x19119a0, 0xc0006a6420, 0x199, 0x4a6c29, 0x19119a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:379 +0x126

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00089be90, 0x15, 0x19119a0, 0xc0006a6420, 0x199, 0x10, 0x1708de0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.pointerFromUnstructured(0x170f560, 0xc00089be90, 0x15, 0x18b8f00, 0xc0004330c8, 0x196, 0xc0005bf180, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:368 +0x2b4

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00089be90, 0x15, 0x18b8f00, 0xc0004330c8, 0x196, 0x1708de0, 0xc0008003a0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:226 +0x701

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00089be30, 0x15, 0x193c3e0, 0xc000432fc0, 0x199, 0xc000b2f0e0, 0x4032df)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x1708de0, 0xc000c8f240, 0x194, 0x193c3e0, 0xc000432fc0, 0x199, 0x199, 0x4a6933)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.sliceFromUnstructured(0x167d1c0, 0xc0007d6780, 0x97, 0x167a900, 0xc0002fb118, 0x197, 0xc0005bf538, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:349 +0x663

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x167d1c0, 0xc0007d6780, 0x97, 0x167a900, 0xc0002fb118, 0x197, 0x1708de0, 0xc000800330)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:224 +0x685

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00089bdd0, 0x15, 0x1923380, 0xc0002fb118, 0x199, 0xc0005bf6f8, 0x413b0f)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00089bdd0, 0x15, 0x1923380, 0xc0002fb118, 0x199, 0x1708de0, 0xc000800320)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.structFromUnstructured(0x170f560, 0xc00089bd10, 0x15, 0x1820100, 0xc0002fb000, 0x199, 0x50868dd79f401bef, 0x502163faec192655)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:390 +0x344

k8s.io/apimachinery/pkg/runtime.fromUnstructured(0x170f560, 0xc00089bd10, 0x15, 0x1820100, 0xc0002fb000, 0x199, 0x203000, 0x203000)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:228 +0x609

k8s.io/apimachinery/pkg/runtime.(*unstructuredConverter).FromUnstructured(0x2886b60, 0xc00089bd10, 0x1942160, 0xc0002fb000, 0x5fb56a0dde79e557, 0xc00089bda0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/converter.go:131 +0x2b8

github.com/argoproj/argo/workflow/util.FromUnstructured(0xc000cc2140, 0xc0003dbef0, 0xc000bf3f00, 0x1b)

	/go/src/github.com/argoproj/argo/workflow/util/util.go:127 +0x69

github.com/argoproj/argo/workflow/ttlcontroller.(*Controller).enqueueWF(0xc00003a460, 0x193f940, 0xc000cc2140)

	/go/src/github.com/argoproj/argo/workflow/ttlcontroller/ttlcontroller.go:125 +0x61

k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:195

k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd(0x1a05570, 0x1bea840, 0xc00000da00, 0x193f940, 0xc000cc2140)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:227 +0x69

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:607 +0x218

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0005bfdd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00049e740)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005bff40, 0xdf8475800, 0x0, 0x1, 0xc000506540)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501a00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0c20)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 156 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501a00)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000dfe90, 0xc0003e0c30)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 170 [select]:

k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000696140, 0x1bbe340, 0xc000868600, 0xc0005bdc00, 0xc00054ce40, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:329 +0x1ec

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000696140, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:300 +0x833

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000591ef0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005bdef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000696140, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0004e4080, 0xc00072a140)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 144 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc000696140, 0xc000622420, 0xc000ccc300, 0xc00054ce40)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 142 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000702dd8, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000593f40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000702f40, 0xdf8475800, 0x0, 0x42db01, 0xc0000aade0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000494200)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000147f70, 0xc000732970)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 158 [chan receive]:

k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000147f10, 0xc0005065a0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:478 +0x49

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea310, 0xc0003ad460)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 159 [chan receive]:

k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000622420, 0xc000450880)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:103 +0x34

created by k8s.io/client-go/tools/cache.(*controller).Run

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:102 +0xae



goroutine 160 [select]:

golang.org/x/net/http2.(*ClientConn).roundTrip(0xc000482600, 0xc0000efa00, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1096 +0x9a0

golang.org/x/net/http2.(*Transport).RoundTripOpt(0xc0002ae060, 0xc0000efa00, 0xc00071ad00, 0x754025, 0x0, 0xc00018d200)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:448 +0x172

golang.org/x/net/http2.(*Transport).RoundTrip(...)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:409

golang.org/x/net/http2.noDialH2RoundTripper.RoundTrip(0xc0002ae060, 0xc0000efa00, 0xc001148380, 0x5, 0xc00018d288)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2613 +0x3e

net/http.(*Transport).roundTrip(0xc0004aca00, 0xc0000efa00, 0x16c7600, 0xc001170e01, 0xc0006a1a20)

	/usr/local/go/src/net/http/transport.go:485 +0xdc1

net/http.(*Transport).RoundTrip(0xc0004aca00, 0xc0000efa00, 0x1954ebc, 0xd, 0xc0006a1ad8)

	/usr/local/go/src/net/http/roundtrip.go:17 +0x35

k8s.io/client-go/transport.(*bearerAuthRoundTripper).RoundTrip(0xc0004a6f90, 0xc0000ef900, 0x1951e42, 0xa, 0xc0006a1960)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:317 +0x266

k8s.io/client-go/transport.(*userAgentRoundTripper).RoundTrip(0xc0001fdde0, 0xc0000ef800, 0xc0001fdde0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/transport/round_trippers.go:167 +0x1c0

net/http.send(0xc0000ef800, 0x1bacd40, 0xc0001fdde0, 0x0, 0x0, 0x0, 0xc0008e60f0, 0x5c8036, 0x1, 0x0)

	/usr/local/go/src/net/http/client.go:250 +0x443

net/http.(*Client).send(0xc0004a6fc0, 0xc0000ef800, 0x0, 0x0, 0x0, 0xc0008e60f0, 0x0, 0x1, 0xc0003108a0)

	/usr/local/go/src/net/http/client.go:174 +0xfa

net/http.(*Client).do(0xc0004a6fc0, 0xc0000ef800, 0x0, 0x0, 0x0)

	/usr/local/go/src/net/http/client.go:641 +0x3ce

net/http.(*Client).Do(0xc0004a6fc0, 0xc0000ef800, 0x0, 0x3, 0xc001148380)

	/usr/local/go/src/net/http/client.go:509 +0x35

k8s.io/client-go/rest.(*Request).WatchWithSpecificDecoders(0xc000463b00, 0xc0006e97c8, 0x7f80173db008, 0x28ad300, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/request.go:573 +0x1e2

k8s.io/client-go/dynamic.(*dynamicResourceClient).Watch(0xc000614870, 0x0, 0x0, 0x0, 0x0, 0xc00003e0a0, 0x50, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/dynamic/simple.go:309 +0x8d0

github.com/argoproj/argo/util/unstructured.NewFilteredUnstructuredInformer.func2(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000cba578, ...)

	/go/src/github.com/argoproj/argo/util/unstructured/unstructured.go:39 +0x1e9

k8s.io/client-go/tools/cache.(*ListWatch).Watch(0xc0001fde60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/listwatch.go:113 +0x6c

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00029dc20, 0xc000622420, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:275 +0x420

k8s.io/client-go/tools/cache.(*Reflector).Run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:124 +0x33

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0005926f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006e9ef0, 0x3b9aca00, 0x0, 0x1, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00029dc20, 0xc000622420)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:123 +0x169

k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:54 +0x2e

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ea330, 0xc0003ad500)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 118 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0006d0dd8, 0x0, 0xc00006cdf8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00006cf40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d0f40, 0xdf8475800, 0x0, 0x1, 0xc0000aad20)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000501800)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00012a0d0, 0xc0003a63e0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 119 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000501800)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00012a0d0, 0xc0003a63f0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 185 [select]:

k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00029dc20, 0xc000622420, 0xc0005ff320, 0xc00054c480)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:235 +0x178

created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229 +0x260



goroutine 120 [chan receive]:

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x194bc17, 0x1)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:602 +0x54

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0006d2dd8, 0x0, 0xc00006cdf8)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:601 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00006cf40)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5e

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0006d2f40, 0xdf8475800, 0x0, 0x1, 0xc0000aad80)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc00037e980)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:599 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000147db0, 0xc0003a6440)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 121 [select]:

k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00037e980)

	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:570 +0x18d

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000147db0, 0xc0003a6450)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x59

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62



goroutine 40 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc000635640, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc000635630)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc000635628, 0xc0009ee400, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc000635600, 0xc0009ee400, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00026d340, 0xc000a08100, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00026d340, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00026d340, 0x1742800, 0xc000a08100, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000806120, 0xc000107000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000da550, 0x0, 0x1bbde40, 0xc0008686c0, 0xc0005600a8, 0x178d980, 0xc0003e0d60, 0xc0005927a0, 0x1307d54)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000a08060, 0x0, 0x194bc17, 0x1, 0x0, 0x1f4, 0x896726)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000868600)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 184 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00026d0c0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00026d0b0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc00026d0a8, 0xc000028800, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc00026d080, 0xc000028800, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc00026d8c0, 0xc00083bee0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc00026d8c0, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc00026d8c0, 0x1742800, 0xc00083bee0, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00086c480, 0xc000106c00, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000da9b0, 0x0, 0x1bbde40, 0xc0005cd980, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00083bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x896726)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0005cd940)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 194 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc00076cca0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc00076cc90)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc00076cc88, 0xc0009ee000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc00076cc60, 0xc0009ee000, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc0006a0580, 0xc00087c0c0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc0006a0580, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc0006a0580, 0x1742800, 0xc00087c0c0, 0x0, 0x0)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00041e300, 0xc0009e6000, 0x400, 0x400, 0x0, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00003a0f0, 0x0, 0x1bbde40, 0xc000868580, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00087c0a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0008684c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe



goroutine 196 [sync.Cond.Wait]:

runtime.goparkunlock(...)

	/usr/local/go/src/runtime/proc.go:310

sync.runtime_notifyListWait(0xc0006354e0, 0x0)

	/usr/local/go/src/runtime/sema.go:510 +0xf8

sync.(*Cond).Wait(0xc0006354d0)

	/usr/local/go/src/sync/cond.go:56 +0x9d

golang.org/x/net/http2.(*pipe).Read(0xc0006354c8, 0xc0009ee200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6

golang.org/x/net/http2.transportResponseBody.Read(0xc0006354a0, 0xc0009ee200, 0x200, 0x200, 0x0, 0x0, 0x0)

	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2032 +0xac

encoding/json.(*Decoder).refill(0xc0004a0580, 0xc00083a160, 0x0)

	/usr/local/go/src/encoding/json/stream.go:161 +0xeb

encoding/json.(*Decoder).readValue(0xc0004a0580, 0x0, 0x0, 0x1725900)

	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc

encoding/json.(*Decoder).Decode(0xc0004a0580, 0x1742800, 0xc00083a160, 0x0, 0xc000498d58)

	/usr/local/go/src/encoding/json/stream.go:63 +0x79

k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000848120, 0xc0009e6400, 0x400, 0x400, 0xc000622600, 0x0, 0x38)

	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:150 +0x1a1

k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000614140, 0x0, 0x1bbde40, 0xc00065a600, 0x0, 0x0, 0x18, 0xc000498df8, 0x12c8b60)

	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89

k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00083a120, 0x0, 0xc00070c000, 0x7f801990c008, 0x0, 0xdf8475800, 0x0)

	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c

k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00065a5c0)

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x175

created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher

	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe


time="2020-07-28T16:14:38Z" level=info msg="config map" name=workflow-controller-configmap

time="2020-07-28T16:14:38Z" level=info msg="Configuration:\nartifactRepository: {}\nmetricsConfig: {}\nnodeEvents: {}\npodSpecLogStrategy: {}\nsso:\n  clientId:\n    key: \"\"\n  clientSecret:\n    key: \"\"\n  issuer: \"\"\n  redirectUrl: \"\"\ntelemetryConfig: {}\n"

time="2020-07-28T16:14:38Z" level=info msg="Persistence configuration disabled"

time="2020-07-28T16:14:38Z" level=info msg="Starting Workflow Controller" version=v2.9.1

time="2020-07-28T16:14:38Z" level=info msg="Workers: workflow: 32, pod: 32"

time="2020-07-28T16:14:38Z" level=info msg="Performing periodic GC every 5m0s"

time="2020-07-28T16:14:38Z" level=info msg="Starting CronWorkflow controller"

time="2020-07-28T16:14:38Z" level=info msg="Persistence disabled - so archived workflow GC disabled - you must restart the controller if you enable this"

time="2020-07-28T16:14:38Z" level=info msg="Starting prometheus metrics server at localhost:9090/metrics"