Azure AD job is getting failed.

[DTG28]

The job is running for more than 8 hours and getting failed with below error.

Incorrect string value: '\xF0\x9F\xA4\x93' for column 'DESCRIPTION' at row 1
Incorrect string value: '\xF0\x9F\x93\xA7Op...' for column 'ENTITLEMENT_VALUE' at row 1

Mapping:
{
"entitlementAttribute": {
"SKU": {
"colsToPropsMap": {
"entitlementID":"skuId~#~char",
"entitlement_value":"skuPartNumber~#~char",
"customproperty1":"appliesTo~#~char",
"customproperty2":"capabilityStatus~#~char",
"customproperty5":"consumedUnits~#~char",
"customproperty7":"prepaidUnits~#~listAsString"
}
},
"ServicePlans": {
"colsToPropsMap": {
"entitlementID":"servicePlanId~#~char",
"entitlement_value":"servicePlanName~#~char",
"customproperty1":"provisioningStatus~#~char",
"customproperty2":"appliesTo~#~char",
"customproperty4":"servicePlanId~#~char"
}
},
"Application": {
"colsToPropsMap": {
"entitlementID":"id~#~char",
"entitlement_value":"displayName~#~char",
"customproperty1":"id~#~bool",
"customproperty2":"resourceAppId~#~bool",
"customproperty4":"orgRestrictions~#~boolListInverse",
"customproperty5":"oauth2AllowImplicitFlow~#~bool",
"customproperty6":"allowPublicClient~#~bool",
"customproperty7":"createdDateTime~#~char"
}
},
"AADGroup": {
"colsToPropsMap": {
"entitlementID" : "id~#~char",
"entitlement_value": "displayName~#~char",
"customproperty1" : "deletionTimestamp~#~char",
"customproperty2" : "description~#~emchar",
"customproperty5" : "dirSyncEnabled~#~char",
"customproperty6" : "lastDirSyncTime~#~char",
"customproperty7" : "mail~#~char",
"customproperty8" : "mailEnabled~#~char",
"customproperty9" : "onPremisesSecurityIdentifier~#~char",
"customproperty10" : "securityEnabled~#~char",
"customproperty11" : "groupTypes~#~listAsString",
"customproperty3" : "membershipRule~#~char",
"customproperty13" :"membershipRuleProcessingState~#~char",
"customproperty16" :"resourceProvisioningOptions~#~char"
}
}
}
}

[stalluri · Accepted solution]

Please validate with below mapping.
{

"entitlementAttribute": {

"SKU": {

"colsToPropsMap": {

"entitlementID":"skuId~#~char",

"entitlement_value":"skuPartNumber~#~emchar",

"customproperty1":"appliesTo~#~char",

"customproperty2":"capabilityStatus~#~char",

"customproperty5":"consumedUnits~#~char",

"customproperty7":"prepaidUnits~#~listAsString"

}

},

"ServicePlans": {

"colsToPropsMap": {

"entitlementID":"servicePlanId~#~char",

"entitlement_value":"servicePlanName~#~emchar",

"customproperty1":"provisioningStatus~#~char",

"customproperty2":"appliesTo~#~char",

"customproperty4":"servicePlanId~#~char"

}

},

"Application": {

"colsToPropsMap": {

"entitlementID":"id~#~char",

"entitlement_value":"displayName~#~emchar",

"customproperty1":"id~#~bool",

"customproperty2":"resourceAppId~#~bool",

"customproperty4":"orgRestrictions~#~boolListInverse",

"customproperty5":"oauth2AllowImplicitFlow~#~bool",

"customproperty6":"allowPublicClient~#~bool",

"customproperty7":"createdDateTime~#~char"

}

},

"AADGroup": {

"colsToPropsMap": {

"entitlementID" : "id~#~char",

"entitlement_value": "displayName~#~emchar",

"customproperty1" : "deletionTimestamp~#~emchar",

"customproperty2" : "description~#~emchar",

"customproperty5" : "dirSyncEnabled~#~char",

"customproperty6" : "lastDirSyncTime~#~char",

"customproperty7" : "mail~#~char",

"customproperty8" : "mailEnabled~#~char",

"customproperty9" : "onPremisesSecurityIdentifier~#~char",

"customproperty10" : "securityEnabled~#~char",

"customproperty11" : "groupTypes~#~listAsString",

"customproperty3" : "membershipRule~#~char",

"customproperty13" :"membershipRuleProcessingState~#~char",

"customproperty16" :"resourceProvisioningOptions~#~char"

}

}

}

}

 If this following error message is displayed in the debug log file:

 Incorrect string value: '\xF0\x9F\x93\...' for column 'ENTITLEMENT_VALUE' at row 1

Incorrect string value: '\xF0\x9F\xA4\x93' for column 'DESCRIPTION' at row 1

 Cause: This error occurs while importing an entitlement that contains an emoji character in the attribute or entitlement name.

 Resolution: If you have the database access, execute the following SQL script in the database: INSERT INTO CONFIGURATION (NAME, CONFIGDATA, DEPRECATED) VALUES ('REST_UNICODE_FILTER', '1', 0);.
Create a ticket to Support for this change.


Solution to reduce the job time.

After the above changes, if you still see the job is taking more time.

Please use the below config to filter the data while importing.

you can use this filter at job level:

{

 "importEntTypes": {

   "AADGroup": {},

   "DirectoryRole": {},

   "SKU": {}

 },

 "excludeEntTypes": {

   "ApplicationInstance": {},

  "Team": {},

   "Channel": {},

   "MemberPermission": {},

   "GuestPermission": {},

   "Subscription": {},

   "Application": {},

   "DirectoryRoleMember": {},

   "ServicePlans": {}

 }

 }

[Saathvik]

@DTG28 : This error generally happens when an unsupported character is there, Incase of Azure we often see this issue appears because of emoji characters. To crop such emoji characters while mapping attributes use emchar as data type instead regular char.

So please replace all your entitlement_value mappings with emchar datatype instead of char  datatype and see if that fixes the issue

[DTG28]

@Saathvik 
Yes, that was the reason @stalluri  updated datatype in the mapping I think.

{

"entitlementAttribute": {

"SKU": {

"colsToPropsMap": {

"entitlementID":"skuId~#~char",

"entitlement_value":"skuPartNumber~#~emchar",

"customproperty1":"appliesTo~#~char",

"customproperty2":"capabilityStatus~#~char",

"customproperty5":"consumedUnits~#~char",

"customproperty7":"prepaidUnits~#~listAsString"

}

},

"ServicePlans": {

"colsToPropsMap": {

"entitlementID":"servicePlanId~#~char",

"entitlement_value":"servicePlanName~#~emchar",

"customproperty1":"provisioningStatus~#~char",

"customproperty2":"appliesTo~#~char",

"customproperty4":"servicePlanId~#~char"

}

},

"Application": {

"colsToPropsMap": {

"entitlementID":"id~#~char",

"entitlement_value":"displayName~#~emchar",

"customproperty1":"id~#~bool",

"customproperty2":"resourceAppId~#~bool",

"customproperty4":"orgRestrictions~#~boolListInverse",

"customproperty5":"oauth2AllowImplicitFlow~#~bool",

"customproperty6":"allowPublicClient~#~bool",

"customproperty7":"createdDateTime~#~char"

}

},

"AADGroup": {

"colsToPropsMap": {

"entitlementID" : "id~#~char",

"entitlement_value": "displayName~#~emchar",

"customproperty1" : "deletionTimestamp~#~emchar",

"customproperty2" : "description~#~emchar",

"customproperty5" : "dirSyncEnabled~#~char",

"customproperty6" : "lastDirSyncTime~#~char",

"customproperty7" : "mail~#~char",

"customproperty8" : "mailEnabled~#~char",

"customproperty9" : "onPremisesSecurityIdentifier~#~char",

"customproperty10" : "securityEnabled~#~char",

"customproperty11" : "groupTypes~#~listAsString",

"customproperty3" : "membershipRule~#~char",

"customproperty13" :"membershipRuleProcessingState~#~char",

"customproperty16" :"resourceProvisioningOptions~#~char"

}

}

}

}

I updated the job filter to below and my job got completed in 10 mins.

{
  "importEntTypes": {
    "AADGroup": {}
  },
  "excludeEntTypes": {
    "ApplicationInstance": {},

    "Team": {},
    "Channel": {},
    "MemberPermission": {},
    "GuestPermission": {},
    "DirectoryRole": {},
    "Subscription": {},
    "Application": {},
    "DirectoryRoleMember": {},
    "SKU": {},
    "ServicePlans": {}
  }
}