[XPU] subgraph support device param copy (#51876)

paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc

@@ -227,13 +227,16 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToXpu(Argument *argument) {
   platform::CPUPlace cpu_place;
   platform::Place xpu_place = platform::XPUPlace(argument->xpu_device_id());
   auto *scope = argument->scope_ptr();
-  framework::ir::Graph &graph = argument->main_graph();
+  framework::ir::Graph &main_graph = argument->main_graph();
 
-  for (auto *node : graph.Nodes()) {
+  for (size_t i = 0; i < main_graph.SubGraphsSize(); i++) {
+    auto *graph = main_graph.GetSubGraph(i);
+    for (auto *node : graph->Nodes()) {
       if (!node->IsVar() || !node->Var()->Persistable()) continue;
       auto *var = scope->FindVar(node->Name());
       if (!var->IsType<phi::DenseTensor>()) continue;
       auto *tensor = var->GetMutable<phi::DenseTensor>();
+      if (tensor->place().GetType() == phi::AllocationType::XPU) continue;
 
       phi::DenseTensor temp_tensor;
       temp_tensor.Resize(tensor->dims());
@@ -241,6 +244,7 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToXpu(Argument *argument) {
       tensor->clear();
       paddle::framework::TensorCopySync(temp_tensor, xpu_place, tensor);
     }
+  }
 }
 #endif