Controller

No

KubeCli kubernetes.Interface\tApiCli apiextensionsclient.Interface\t\ttrainingjobLister paddlelisters.TrainingJobLister\ttrainingjobSynced cache.InformerSynced\tjobtracker *sync.Map\tworkqueue workqueue.RateLimitingInterface\trecorder record.EventRecorder\tautoclean bool

c.workqueue.Done(key)

c.workqueue.AddRateLimited(key)

Starting trainingjob controller

Error syncing job

Err

Yes

Successfully synced

createCRD()

Controller

put into workqueue again

forget

Err== nil

WaitForCacheSync()

jobUpdater.UID() != job.ObjectMeta.UID

jobUpdater.Update(job)

job

return true

jobUpdaterObj

jobtracker:(job full name : job updater)

c.workqueue.Forget(key)

gc.cleanOrphanReplicaSets()# delete \"paddle-job-master\" and \"paddle-job-pserver\"gc.cleanOrphanBatchJobs()# delete \"paddle-job\"gc.CleanGarbagePods()

currentPhase = jobUpdater.GetJob().Status.Phase

True

wait.Until()

jobIsDeleted = false

Deleting TrainingJob

jobUpdater.Delete()

Fail

invalid key

currentPhase == Creating || running

jobUpdaterObj - JobUpdater

key

IsNotFound(Err)

OK

Err?

jobUpdater.Reconcile()

Starting workers

c.jobtracker.Delete(key)

False

jobIsDeleted = true

processNextWorkItem()

Exists?

c.workqueue.Get()

jobIsDeleted

NewGarbageCollector()CleanOrphans()