579746 XSD validation not correct / nilable groups

* xmlschemas.c: when a particle need to be processed via counted
  transition, if the group is nillable, the counting won't work, so
  keep track of nillable subset as they are built and generate the
  appropriate epsilon transitions as needed
* test/schemas/579746* result/schemas/579746*: add related test cases
  based on the bug report
diff --git a/xmlschemas.c b/xmlschemas.c
index 96d55b8..8db448c 100644
--- a/xmlschemas.c
+++ b/xmlschemas.c
@@ -12525,7 +12525,12 @@
  *									*
  ************************************************************************/
 
-static void
+/**
+ * xmlSchemaBuildContentModelForSubstGroup:
+ *
+ * Returns 1 if nillable, 0 otherwise
+ */
+static int
 xmlSchemaBuildContentModelForSubstGroup(xmlSchemaParserCtxtPtr pctxt,
 	xmlSchemaParticlePtr particle, int counter, xmlAutomataStatePtr end)
 {
@@ -12533,6 +12538,7 @@
     xmlSchemaElementPtr elemDecl, member;
     xmlSchemaSubstGroupPtr substGroup;
     int i;
+    int ret = 0;
 
     elemDecl = (xmlSchemaElementPtr) particle->children;
     /*
@@ -12548,7 +12554,7 @@
 	    "Internal error: xmlSchemaBuildContentModelForSubstGroup, "
 	    "declaration is marked having a subst. group but none "
 	    "available.\n", elemDecl->name, NULL);
-	return;
+	return(0);
     }
     if (counter >= 0) {
 	/*
@@ -12628,21 +12634,31 @@
 	xmlAutomataNewCountedTrans(pctxt->am, hop, start, counter);
 	xmlAutomataNewCounterTrans(pctxt->am, hop, end, counter);
     }
-    if (particle->minOccurs == 0)
+    if (particle->minOccurs == 0) {
 	xmlAutomataNewEpsilon(pctxt->am, start, end);
+        ret = 1;
+    }
     pctxt->state = end;
+    return(ret);
 }
 
-static void
+/**
+ * xmlSchemaBuildContentModelForElement:
+ *
+ * Returns 1 if nillable, 0 otherwise
+ */
+static int
 xmlSchemaBuildContentModelForElement(xmlSchemaParserCtxtPtr ctxt,
 				     xmlSchemaParticlePtr particle)
 {
+    int ret = 0;
+
     if (((xmlSchemaElementPtr) particle->children)->flags &
 	XML_SCHEMAS_ELEM_SUBST_GROUP_HEAD) {
 	/*
 	* Substitution groups.
 	*/
-	xmlSchemaBuildContentModelForSubstGroup(ctxt, particle, -1, NULL);
+	ret = xmlSchemaBuildContentModelForSubstGroup(ctxt, particle, -1, NULL);
     } else {
 	xmlSchemaElementPtr elemDecl;
 	xmlAutomataStatePtr start;
@@ -12650,7 +12666,7 @@
 	elemDecl = (xmlSchemaElementPtr) particle->children;
 
 	if (elemDecl->flags & XML_SCHEMAS_ELEM_ABSTRACT)
-	    return;
+	    return(0);
 	if (particle->maxOccurs == 1) {
 	    start = ctxt->state;
 	    ctxt->state = xmlAutomataNewTransition2(ctxt->am, start, NULL,
@@ -12678,9 +12694,12 @@
 	    ctxt->state = xmlAutomataNewCounterTrans(ctxt->am, ctxt->state,
 		NULL, counter);
 	}
-	if (particle->minOccurs == 0)
+	if (particle->minOccurs == 0) {
 	    xmlAutomataNewEpsilon(ctxt->am, start, ctxt->state);
+            ret = 1;
+        }
     }
+    return(ret);
 }
 
 /**
@@ -12691,21 +12710,24 @@
  *
  * Create the automaton for the {content type} of a complex type.
  *
+ * Returns 1 if the content is nillable, 0 otherwise
  */
-static void
+static int
 xmlSchemaBuildAContentModel(xmlSchemaParserCtxtPtr pctxt,
 			    xmlSchemaParticlePtr particle)
 {
+    int ret = 0, tmp2;
+
     if (particle == NULL) {
 	PERROR_INT("xmlSchemaBuildAContentModel", "particle is NULL");
-	return;
+	return(1);
     }
     if (particle->children == NULL) {
 	/*
 	* Just return in this case. A missing "term" of the particle
 	* might arise due to an invalid "term" component.
 	*/
-	return;
+	return(1);
     }
 
     switch (particle->children->type) {
@@ -12757,7 +12779,8 @@
 		int counter;
 		xmlAutomataStatePtr hop;
 		int maxOccurs =
-		    particle->maxOccurs == UNBOUNDED ? UNBOUNDED : particle->maxOccurs - 1;
+		    particle->maxOccurs == UNBOUNDED ? UNBOUNDED :
+                                           particle->maxOccurs - 1;
 		int minOccurs =
 		    particle->minOccurs < 1 ? 0 : particle->minOccurs - 1;
 
@@ -12792,247 +12815,273 @@
 	    }
 	    if (particle->minOccurs == 0) {
 		xmlAutomataNewEpsilon(pctxt->am, start, end);
+                ret = 1;
 	    }
 	    pctxt->state = end;
             break;
 	}
         case XML_SCHEMA_TYPE_ELEMENT:
-	    xmlSchemaBuildContentModelForElement(pctxt, particle);
+	    ret = xmlSchemaBuildContentModelForElement(pctxt, particle);
 	    break;
         case XML_SCHEMA_TYPE_SEQUENCE:{
-                xmlSchemaTreeItemPtr sub;
+            xmlSchemaTreeItemPtr sub;
 
-                /*
-                 * If max and min occurances are default (1) then
-                 * simply iterate over the particles of the <sequence>.
-                 */
-                if ((particle->minOccurs == 1) && (particle->maxOccurs == 1)) {
-                    sub = particle->children->children;
-                    while (sub != NULL) {
-                        xmlSchemaBuildAContentModel(pctxt,
-			    (xmlSchemaParticlePtr) sub);
-                        sub = sub->next;
-                    }
-                } else {
-                    xmlAutomataStatePtr oldstate = pctxt->state;
+            ret = 1;
+            /*
+             * If max and min occurances are default (1) then
+             * simply iterate over the particles of the <sequence>.
+             */
+            if ((particle->minOccurs == 1) && (particle->maxOccurs == 1)) {
+                sub = particle->children->children;
 
-                    if (particle->maxOccurs >= UNBOUNDED) {
-                        if (particle->minOccurs > 1) {
-                            xmlAutomataStatePtr tmp;
-                            int counter;
+                while (sub != NULL) {
+                    tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                    if (tmp2 != 1) ret = 0;
+                    sub = sub->next;
+                }
+            } else {
+                xmlAutomataStatePtr oldstate = pctxt->state;
 
-                            pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
-				oldstate, NULL);
-                            oldstate = pctxt->state;
-
-                            counter = xmlAutomataNewCounter(pctxt->am,
-				particle->minOccurs - 1, UNBOUNDED);
-
-                            sub = particle->children->children;
-                            while (sub != NULL) {
-                                xmlSchemaBuildAContentModel(pctxt,
-				    (xmlSchemaParticlePtr) sub);
-                                sub = sub->next;
-                            }
-                            tmp = pctxt->state;
-                            xmlAutomataNewCountedTrans(pctxt->am, tmp,
-                                                       oldstate, counter);
-                            pctxt->state =
-                                xmlAutomataNewCounterTrans(pctxt->am, tmp,
-                                                           NULL, counter);
-
-                        } else {
-                            pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
-				oldstate, NULL);
-                            oldstate = pctxt->state;
-
-			    sub = particle->children->children;
-                            while (sub != NULL) {
-                                xmlSchemaBuildAContentModel(pctxt,
-				    (xmlSchemaParticlePtr) sub);
-                                sub = sub->next;
-                            }
-                            xmlAutomataNewEpsilon(pctxt->am, pctxt->state,
-                                                  oldstate);
-			    /*
-			     * epsilon needed to block previous trans from
-			     * being allowed to enter back from another
-			     * construct
-			     */
-			    pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
-						pctxt->state, NULL);
-                            if (particle->minOccurs == 0) {
-                                xmlAutomataNewEpsilon(pctxt->am,
-				    oldstate, pctxt->state);
-                            }
-                        }
-                    } else if ((particle->maxOccurs > 1)
-                               || (particle->minOccurs > 1)) {
+                if (particle->maxOccurs >= UNBOUNDED) {
+                    if (particle->minOccurs > 1) {
                         xmlAutomataStatePtr tmp;
                         int counter;
 
                         pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
-			    oldstate, NULL);
+                            oldstate, NULL);
                         oldstate = pctxt->state;
 
                         counter = xmlAutomataNewCounter(pctxt->am,
-			    particle->minOccurs - 1,
-			    particle->maxOccurs - 1);
+                            particle->minOccurs - 1, UNBOUNDED);
 
                         sub = particle->children->children;
                         while (sub != NULL) {
-                            xmlSchemaBuildAContentModel(pctxt,
-				(xmlSchemaParticlePtr) sub);
+                            tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                            (xmlSchemaParticlePtr) sub);
+                            if (tmp2 != 1) ret = 0;
                             sub = sub->next;
                         }
                         tmp = pctxt->state;
-                        xmlAutomataNewCountedTrans(pctxt->am,
-			    tmp, oldstate, counter);
+                        xmlAutomataNewCountedTrans(pctxt->am, tmp,
+                                                   oldstate, counter);
                         pctxt->state =
-                            xmlAutomataNewCounterTrans(pctxt->am, tmp, NULL,
-                                                       counter);
-                        if (particle->minOccurs == 0) {
+                            xmlAutomataNewCounterTrans(pctxt->am, tmp,
+                                                       NULL, counter);
+                        if (ret == 1)
                             xmlAutomataNewEpsilon(pctxt->am,
-				oldstate, pctxt->state);
-                        }
+                                                oldstate, pctxt->state);
+
                     } else {
+                        pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
+                            oldstate, NULL);
+                        oldstate = pctxt->state;
+
                         sub = particle->children->children;
                         while (sub != NULL) {
-                            xmlSchemaBuildAContentModel(pctxt,
-				(xmlSchemaParticlePtr) sub);
+                            tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                            if (tmp2 != 1) ret = 0;
                             sub = sub->next;
                         }
+                        xmlAutomataNewEpsilon(pctxt->am, pctxt->state,
+                                              oldstate);
+                        /*
+                         * epsilon needed to block previous trans from
+                         * being allowed to enter back from another
+                         * construct
+                         */
+                        pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
+                                            pctxt->state, NULL);
                         if (particle->minOccurs == 0) {
-                            xmlAutomataNewEpsilon(pctxt->am, oldstate,
-                                                  pctxt->state);
+                            xmlAutomataNewEpsilon(pctxt->am,
+                                oldstate, pctxt->state);
+                            ret = 1;
                         }
                     }
-                }
-                break;
-            }
-        case XML_SCHEMA_TYPE_CHOICE:{
-                xmlSchemaTreeItemPtr sub;
-                xmlAutomataStatePtr start, end;
+                } else if ((particle->maxOccurs > 1)
+                           || (particle->minOccurs > 1)) {
+                    xmlAutomataStatePtr tmp;
+                    int counter;
 
-                start = pctxt->state;
-                end = xmlAutomataNewState(pctxt->am);
+                    pctxt->state = xmlAutomataNewEpsilon(pctxt->am,
+                        oldstate, NULL);
+                    oldstate = pctxt->state;
 
-                /*
-                 * iterate over the subtypes and remerge the end with an
-                 * epsilon transition
-                 */
-                if (particle->maxOccurs == 1) {
-		    sub = particle->children->children;
+                    counter = xmlAutomataNewCounter(pctxt->am,
+                        particle->minOccurs - 1,
+                        particle->maxOccurs - 1);
+
+                    sub = particle->children->children;
                     while (sub != NULL) {
-                        pctxt->state = start;
-                        xmlSchemaBuildAContentModel(pctxt,
-			    (xmlSchemaParticlePtr) sub);
-                        xmlAutomataNewEpsilon(pctxt->am, pctxt->state, end);
+                        tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                        if (tmp2 != 1) ret = 0;
                         sub = sub->next;
                     }
+                    tmp = pctxt->state;
+                    xmlAutomataNewCountedTrans(pctxt->am,
+                        tmp, oldstate, counter);
+                    pctxt->state =
+                        xmlAutomataNewCounterTrans(pctxt->am, tmp, NULL,
+                                                   counter);
+                    if ((particle->minOccurs == 0) || (ret == 1)) {
+                        xmlAutomataNewEpsilon(pctxt->am,
+                                            oldstate, pctxt->state);
+                        ret = 1;
+                    }
                 } else {
-                    int counter;
-                    xmlAutomataStatePtr hop, base;
-                    int maxOccurs = particle->maxOccurs == UNBOUNDED ?
-                        UNBOUNDED : particle->maxOccurs - 1;
-                    int minOccurs =
-                        particle->minOccurs < 1 ? 0 : particle->minOccurs - 1;
-
-                    /*
-                     * use a counter to keep track of the number of transtions
-                     * which went through the choice.
-                     */
-                    counter =
-                        xmlAutomataNewCounter(pctxt->am, minOccurs, maxOccurs);
-                    hop = xmlAutomataNewState(pctxt->am);
-                    base = xmlAutomataNewState(pctxt->am);
-
-		    sub = particle->children->children;
+                    sub = particle->children->children;
                     while (sub != NULL) {
-                        pctxt->state = base;
-                        xmlSchemaBuildAContentModel(pctxt,
-			    (xmlSchemaParticlePtr) sub);
-                        xmlAutomataNewEpsilon(pctxt->am, pctxt->state, hop);
+                        tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                        if (tmp2 != 1) ret = 0;
                         sub = sub->next;
                     }
-                    xmlAutomataNewEpsilon(pctxt->am, start, base);
-		    xmlAutomataNewCountedTrans(pctxt->am, hop, base, counter);
-                    xmlAutomataNewCounterTrans(pctxt->am, hop, end, counter);
+                    if (particle->minOccurs == 0) {
+                        xmlAutomataNewEpsilon(pctxt->am, oldstate,
+                                              pctxt->state);
+                        ret = 1;
+                    }
                 }
-                if (particle->minOccurs == 0) {
-                    xmlAutomataNewEpsilon(pctxt->am, start, end);
-                }
-                pctxt->state = end;
-                break;
             }
-        case XML_SCHEMA_TYPE_ALL:{
-                xmlAutomataStatePtr start, tmp;
-		xmlSchemaParticlePtr sub;
-		xmlSchemaElementPtr elemDecl;
+            break;
+        }
+        case XML_SCHEMA_TYPE_CHOICE:{
+            xmlSchemaTreeItemPtr sub;
+            xmlAutomataStatePtr start, end;
 
-		sub = (xmlSchemaParticlePtr) particle->children->children;
-                if (sub == NULL)
-                    break;
-                start = pctxt->state;
-                tmp = xmlAutomataNewState(pctxt->am);
-                xmlAutomataNewEpsilon(pctxt->am, pctxt->state, tmp);
-                pctxt->state = tmp;
+            ret = 0;
+            start = pctxt->state;
+            end = xmlAutomataNewState(pctxt->am);
+
+            /*
+             * iterate over the subtypes and remerge the end with an
+             * epsilon transition
+             */
+            if (particle->maxOccurs == 1) {
+                sub = particle->children->children;
                 while (sub != NULL) {
-                    pctxt->state = tmp;
-
-		    elemDecl = (xmlSchemaElementPtr) sub->children;
-		    if (elemDecl == NULL) {
-			PERROR_INT("xmlSchemaBuildAContentModel",
-			    "<element> particle has no term");
-			return;
-		    };
-		    /*
-		    * NOTE: The {max occurs} of all the particles in the
-		    * {particles} of the group must be 0 or 1; this is
-		    * already ensured during the parse of the content of
-		    * <all>.
-		    */
-		    if (elemDecl->flags & XML_SCHEMAS_ELEM_SUBST_GROUP_HEAD) {
-			int counter;
-
-		        /*
-			 * This is an abstract group, we need to share
-			 * the same counter for all the element transitions
-			 * derived from the group
-			 */
-			counter = xmlAutomataNewCounter(pctxt->am,
-			                   sub->minOccurs, sub->maxOccurs);
-			xmlSchemaBuildContentModelForSubstGroup(pctxt,
-					   sub, counter, pctxt->state);
-		    } else {
-			if ((sub->minOccurs == 1) &&
-			    (sub->maxOccurs == 1)) {
-			    xmlAutomataNewOnceTrans2(pctxt->am, pctxt->state,
-						    pctxt->state,
-						    elemDecl->name,
-						    elemDecl->targetNamespace,
-						    1, 1, elemDecl);
-			} else if ((sub->minOccurs == 0) &&
-			    (sub->maxOccurs == 1)) {
-
-			    xmlAutomataNewCountTrans2(pctxt->am, pctxt->state,
-						     pctxt->state,
-						     elemDecl->name,
-						     elemDecl->targetNamespace,
-						     0,
-						     1,
-						     elemDecl);
-			}
-		    }
-                    sub = (xmlSchemaParticlePtr) sub->next;
+                    pctxt->state = start;
+                    tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                    if (tmp2 == 1) ret = 1;
+                    xmlAutomataNewEpsilon(pctxt->am, pctxt->state, end);
+                    sub = sub->next;
                 }
-                pctxt->state =
-                    xmlAutomataNewAllTrans(pctxt->am, pctxt->state, NULL, 0);
-                if (particle->minOccurs == 0) {
-                    xmlAutomataNewEpsilon(pctxt->am, start, pctxt->state);
+            } else {
+                int counter;
+                xmlAutomataStatePtr hop, base;
+                int maxOccurs = particle->maxOccurs == UNBOUNDED ?
+                    UNBOUNDED : particle->maxOccurs - 1;
+                int minOccurs =
+                    particle->minOccurs < 1 ? 0 : particle->minOccurs - 1;
+
+                /*
+                 * use a counter to keep track of the number of transtions
+                 * which went through the choice.
+                 */
+                counter =
+                    xmlAutomataNewCounter(pctxt->am, minOccurs, maxOccurs);
+                hop = xmlAutomataNewState(pctxt->am);
+                base = xmlAutomataNewState(pctxt->am);
+
+                sub = particle->children->children;
+                while (sub != NULL) {
+                    pctxt->state = base;
+                    tmp2 = xmlSchemaBuildAContentModel(pctxt,
+                                        (xmlSchemaParticlePtr) sub);
+                    if (tmp2 == 1) ret = 1;
+                    xmlAutomataNewEpsilon(pctxt->am, pctxt->state, hop);
+                    sub = sub->next;
                 }
-                break;
+                xmlAutomataNewEpsilon(pctxt->am, start, base);
+                xmlAutomataNewCountedTrans(pctxt->am, hop, base, counter);
+                xmlAutomataNewCounterTrans(pctxt->am, hop, end, counter);
+                if (ret == 1)
+                    xmlAutomataNewEpsilon(pctxt->am, base, end);
             }
+            if (particle->minOccurs == 0) {
+                xmlAutomataNewEpsilon(pctxt->am, start, end);
+                ret = 1;
+            }
+            pctxt->state = end;
+            break;
+        }
+        case XML_SCHEMA_TYPE_ALL:{
+            xmlAutomataStatePtr start, tmp;
+            xmlSchemaParticlePtr sub;
+            xmlSchemaElementPtr elemDecl;
+
+            ret = 1;
+
+            sub = (xmlSchemaParticlePtr) particle->children->children;
+            if (sub == NULL)
+                break;
+
+            ret = 0;
+
+            start = pctxt->state;
+            tmp = xmlAutomataNewState(pctxt->am);
+            xmlAutomataNewEpsilon(pctxt->am, pctxt->state, tmp);
+            pctxt->state = tmp;
+            while (sub != NULL) {
+                pctxt->state = tmp;
+
+                elemDecl = (xmlSchemaElementPtr) sub->children;
+                if (elemDecl == NULL) {
+                    PERROR_INT("xmlSchemaBuildAContentModel",
+                        "<element> particle has no term");
+                    return(ret);
+                };
+                /*
+                * NOTE: The {max occurs} of all the particles in the
+                * {particles} of the group must be 0 or 1; this is
+                * already ensured during the parse of the content of
+                * <all>.
+                */
+                if (elemDecl->flags & XML_SCHEMAS_ELEM_SUBST_GROUP_HEAD) {
+                    int counter;
+
+                    /*
+                     * This is an abstract group, we need to share
+                     * the same counter for all the element transitions
+                     * derived from the group
+                     */
+                    counter = xmlAutomataNewCounter(pctxt->am,
+                                       sub->minOccurs, sub->maxOccurs);
+                    xmlSchemaBuildContentModelForSubstGroup(pctxt,
+                                       sub, counter, pctxt->state);
+                } else {
+                    if ((sub->minOccurs == 1) &&
+                        (sub->maxOccurs == 1)) {
+                        xmlAutomataNewOnceTrans2(pctxt->am, pctxt->state,
+                                                pctxt->state,
+                                                elemDecl->name,
+                                                elemDecl->targetNamespace,
+                                                1, 1, elemDecl);
+                    } else if ((sub->minOccurs == 0) &&
+                        (sub->maxOccurs == 1)) {
+
+                        xmlAutomataNewCountTrans2(pctxt->am, pctxt->state,
+                                                 pctxt->state,
+                                                 elemDecl->name,
+                                                 elemDecl->targetNamespace,
+                                                 0,
+                                                 1,
+                                                 elemDecl);
+                    }
+                }
+                sub = (xmlSchemaParticlePtr) sub->next;
+            }
+            pctxt->state =
+                xmlAutomataNewAllTrans(pctxt->am, pctxt->state, NULL, 0);
+            if (particle->minOccurs == 0) {
+                xmlAutomataNewEpsilon(pctxt->am, start, pctxt->state);
+                ret = 1;
+            }
+            break;
+        }
 	case XML_SCHEMA_TYPE_GROUP:
 	    /*
 	    * If we hit a model group definition, then this means that
@@ -13041,14 +13090,16 @@
 	    * TODO: But the group should be substituted and not occur at
 	    * all in the content model at this point. Fix this.
 	    */
+            ret = 1;
 	    break;
         default:
 	    xmlSchemaInternalErr2(ACTXT_CAST pctxt,
 		"xmlSchemaBuildAContentModel",
 		"found unexpected term of type '%s' in content model",
 		WXS_ITEM_TYPE_NAME(particle->children), NULL);
-            return;
+            return(ret);
     }
+    return(ret);
 }
 
 /**