Merge pull request kubernetes#22149 from mfojtik/mfojtik-wait-for-crd…

…-discovery

UPSTREAM: 00000: wait for CRD discovery to be successful once before

Origin-commit: 9ffad072ee9a512a7344dba700b5a15bcad5349a

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -21,10 +21,15 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/golang/glog"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
@@ -208,6 +213,52 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		go finalizingController.Run(5, context.StopCh)
 		return nil
 	})
+	s.GenericAPIServer.AddPostStartHookOrDie("crd-discovery-available", func(context genericapiserver.PostStartHookContext) error {
+		return wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
+			// only check if we have a valid list for a given resourceversion
+			if !s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions().Informer().HasSynced() {
+				return false, nil
+			}
+
+			_, serverGroupsAndResources, err := crdClient.Discovery().ServerGroupsAndResources()
+			if err != nil {
+				return false, err
+			}
+
+			serverCRDs, err := s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions().Lister().List(labels.Everything())
+			if err != nil {
+				return false, err
+			}
+
+			crdGroupsAndResources := sets.NewString()
+			for _, crd := range serverCRDs {
+				// Skip not active CRD
+				if !apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) {
+					continue
+				}
+				for _, version := range crd.Spec.Versions {
+					// Skip versions that are not served
+					if !version.Served {
+						continue
+					}
+					crdGroupsAndResources.Insert(fmt.Sprintf("%s.%s.%s", crd.Spec.Names.Plural, version.Name, crd.Spec.Group))
+				}
+			}
+
+			discoveryGroupsAndResources := sets.NewString()
+			for _, resource := range serverGroupsAndResources {
+				for _, apiResource := range resource.APIResources {
+					discoveryGroupsAndResources.Insert(fmt.Sprintf("%s.%s.%s", apiResource.Name, apiResource.Version, apiResource.Group))
+				}
+			}
+
+			if !discoveryGroupsAndResources.HasAll(crdGroupsAndResources.List()...) {
+				glog.Infof("waiting for CRD resources in discovery: %#v", discoveryGroupsAndResources.Difference(crdGroupsAndResources))	
+				return false, nil
+			}
+			return true, nil
+		}, context.StopCh)
+	})
 
 	// we don't want to report healthy until we can handle all CRDs that have already been registered.  Waiting for the informer
 	// to sync makes sure that the lister will be valid before we begin.  There may still be races for CRDs added after startup,

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -74,7 +74,6 @@ func NewDiscoveryController(crdInformer informers.CustomResourceDefinitionInform
 }
 
 func (c *DiscoveryController) sync(version schema.GroupVersion) error {
-
 	apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
 	apiResourcesForDiscovery := []metav1.APIResource{}
 	versionsForDiscoveryMap := map[metav1.GroupVersion]bool{}
@@ -130,6 +129,8 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 			Name:         crd.Status.AcceptedNames.Plural,
 			SingularName: crd.Status.AcceptedNames.Singular,
 			Namespaced:   crd.Spec.Scope == apiextensions.NamespaceScoped,
+			Group:        crd.Spec.Group,
+			Version:      version.Version,
 			Kind:         crd.Status.AcceptedNames.Kind,
 			Verbs:        verbs,
 			ShortNames:   crd.Status.AcceptedNames.ShortNames,
@@ -138,21 +139,29 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 
 		if crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil {
 			apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
-				Name:       crd.Status.AcceptedNames.Plural + "/status",
-				Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
-				Kind:       crd.Status.AcceptedNames.Kind,
-				Verbs:      metav1.Verbs([]string{"get", "patch", "update"}),
+				Name:         crd.Status.AcceptedNames.Plural + "/status",
+				SingularName: "",
+				Namespaced:   crd.Spec.Scope == apiextensions.NamespaceScoped,
+				Group:        crd.Spec.Group,
+				Version:      version.Version,
+				Kind:         crd.Status.AcceptedNames.Kind,
+				Verbs:        metav1.Verbs([]string{"get", "patch", "update"}),
+				ShortNames:   nil,
+				Categories:   nil,
 			})
 		}
 
 		if crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil {
 			apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
-				Group:      autoscaling.GroupName,
-				Version:    "v1",
-				Kind:       "Scale",
-				Name:       crd.Status.AcceptedNames.Plural + "/scale",
-				Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
-				Verbs:      metav1.Verbs([]string{"get", "patch", "update"}),
+				Name:         crd.Status.AcceptedNames.Plural + "/scale",
+				SingularName: "",
+				Namespaced:   crd.Spec.Scope == apiextensions.NamespaceScoped,
+				Group:        autoscaling.GroupName,
+				Version:      "v1",
+				Kind:         "Scale",
+				Verbs:        metav1.Verbs([]string{"get", "patch", "update"}),
+				ShortNames:   nil,
+				Categories:   nil,
 			})
 		}
 	}

staging/src/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -377,6 +377,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			resourcePath = itemPath
 			resourceParams = nameParams
 		}
+		apiResource.Group = a.group.GroupVersion.Group
+		apiResource.Version = a.group.GroupVersion.Version
 		apiResource.Name = path
 		apiResource.Namespaced = false
 		apiResource.Kind = resourceKind
@@ -427,6 +429,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			resourcePath = itemPath
 			resourceParams = nameParams
 		}
+		apiResource.Group = a.group.GroupVersion.Group
+		apiResource.Version = a.group.GroupVersion.Version
 		apiResource.Name = path
 		apiResource.Namespaced = true
 		apiResource.Kind = resourceKind